{
  "page": {
    "id": "kctbh9vrtdwd",
    "name": "GitHub",
    "url": "https://web.archive.org/web/20231206141002/https://www.githubstatus.com"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2023-11-03T19:15:40.357Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2023-11-07T14:25:35.851Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-28T17:59:15.474Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Visit www.githubstatus.com for more information","created_at":"2018-12-05T19:39:40.838Z","updated_at":"2022-09-07T00:08:33.519Z","position":4,"description":null,"showcase":false,"start_date":null,"id":"0l2p9nhqnxpd","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2023-11-27T20:44:01.585Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2023-11-27T20:43:49.050Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-28T17:59:10.423Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Packages","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2023-11-03T19:13:42.721Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":false,"start_date":null,"id":"st3j38cctv9l","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2023-11-11T02:14:00.543Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2023-11-03T19:14:51.990Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2023-12-01T18:16:24.775Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}],
  "incidents": [{"name":"Incident with Copilot","status":"resolved","created_at":"2023-12-01T15:49:14.540Z","updated_at":"2023-12-01T18:16:24.820Z","monitoring_at":null,"resolved_at":"2023-12-01T18:16:24.805Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/80jk9ylc3wk1","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-12-01T15:49:14.534Z","reminder_intervals":null,"id":"fj7v3z6fy7ym","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2023-12-01T18:16:24.805Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-12-01T18:16:24.805Z","display_at":"2023-12-01T18:16:24.805Z","deliver_notifications":true,"tweet_id":null,"id":"fk8dpmdjvtdm","incident_id":"fj7v3z6fy7ym","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"A small percentage of Copilot Chat users are still experiencing long request times and errors. We are still investigating to determine the root cause.","created_at":"2023-12-01T17:36:07.671Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-12-01T17:36:07.671Z","display_at":"2023-12-01T17:36:07.671Z","deliver_notifications":true,"tweet_id":null,"id":"sgnk0ygk11hf","incident_id":"fj7v3z6fy7ym","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Some customers are experiencing higher latency for Copilot Chat. We are continuing our investigation.","created_at":"2023-12-01T16:41:05.149Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-12-01T16:41:05.149Z","display_at":"2023-12-01T16:41:05.149Z","deliver_notifications":true,"tweet_id":null,"id":"kbqdmhv8wxgf","incident_id":"fj7v3z6fy7ym","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Copilot is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-12-01T15:55:41.056Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-12-01T15:55:41.056Z","display_at":"2023-12-01T15:55:41.056Z","deliver_notifications":true,"tweet_id":null,"id":"ycnrkyk6pc1m","incident_id":"fj7v3z6fy7ym","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports that that some customers are experiencing increased latency and failed requests for Copilot Chat.","created_at":"2023-12-01T15:53:10.137Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-12-01T15:53:10.137Z","display_at":"2023-12-01T15:53:10.137Z","deliver_notifications":true,"tweet_id":null,"id":"p2hy5fqtmj3g","incident_id":"fj7v3z6fy7ym","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-12-01T15:49:14.573Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-12-01T15:49:14.573Z","display_at":"2023-12-01T15:49:14.573Z","deliver_notifications":true,"tweet_id":null,"id":"f5jk6bqm5pxp","incident_id":"fj7v3z6fy7ym","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2023-12-01T18:16:24.775Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"We are investigating reports of degraded performance.","status":"resolved","created_at":"2023-11-28T18:09:52.094Z","updated_at":"2023-11-28T19:40:12.239Z","monitoring_at":null,"resolved_at":"2023-11-28T19:40:12.220Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/wlw5pnj5q070","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-28T18:09:52.085Z","reminder_intervals":null,"id":"dq9m4n5tfyr6","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2023-11-28T19:40:12.220Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T19:40:12.220Z","display_at":"2023-11-28T19:40:12.220Z","deliver_notifications":true,"tweet_id":null,"id":"7sfzr95w2sq9","incident_id":"dq9m4n5tfyr6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We were not able to publish webhooks in response to push events triggered between 16:23 and 17:12 UTC. To avoid further disruption to customer workflows, we’ve decided not to continue our attempts to re-process those events. \nWorkflows that trigger on pull_request or git push events may not have been run during this time period. \nYou can run impacted workflows manually or by pushing a new commit to the same branch.","created_at":"2023-11-28T19:36:09.308Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T19:36:09.308Z","display_at":"2023-11-28T19:36:09.308Z","deliver_notifications":true,"tweet_id":null,"id":"jz20w5xjzgp7","incident_id":"dq9m4n5tfyr6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Customers saw push event deliveries for Actions and Webhooks fail between 16:23 and 17:12 (UTC). We fixed the issue, and we are working to re-process push events for the affected time period.","created_at":"2023-11-28T18:09:59.086Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T18:09:59.086Z","display_at":"2023-11-28T18:09:59.086Z","deliver_notifications":true,"tweet_id":null,"id":"bwtv5fms619w","incident_id":"dq9m4n5tfyr6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-11-28T18:09:52.143Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T18:09:52.143Z","display_at":"2023-11-28T18:09:52.143Z","deliver_notifications":true,"tweet_id":null,"id":"8d3x1zqzpjms","incident_id":"dq9m4n5tfyr6","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Incident with Actions and Webhooks","status":"resolved","created_at":"2023-11-28T17:24:53.273Z","updated_at":"2023-11-30T20:37:50.393Z","monitoring_at":null,"resolved_at":"2023-11-28T17:59:15.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/xycyhtpd314p","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-28T17:24:53.265Z","reminder_intervals":null,"id":"dsb3kn1zfdl7","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"This incident has been resolved. An interaction between two feature flag rollouts caused us to suppress delivery of push webhooks between 16:23 and 17:11 UTC in a manner that evaded our existing observability. This affected 71,000 repositories whose users will have noted missing webhook deliveries and/or experienced Actions jobs with push triggers failing to start. During this period, around 25% of new Actions jobs were impacted. The issues were resolved by disabling one of the feature flags in question.\n\nAfter weighing various options for retroactively dispatching old push webhooks, we concluded that the risk of delivering stale data to customers with these redeliveries outweighed the possible benefit.\n\nAs follow up to this incident, we are working to improve monitoring of webhook throughput and to document a policy around webhook redelivery timelines.","created_at":"2023-11-28T17:59:15.499Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-30T20:37:50.376Z","display_at":"2023-11-28T17:59:15.000Z","deliver_notifications":true,"tweet_id":null,"id":"b98cydv1cd9d","incident_id":"dsb3kn1zfdl7","custom_tweet":null,"affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Actions is operating normally.","created_at":"2023-11-28T17:59:10.453Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T17:59:10.453Z","display_at":"2023-11-28T17:59:10.453Z","deliver_notifications":true,"tweet_id":null,"id":"pgwxt2m5kbxj","incident_id":"dsb3kn1zfdl7","custom_tweet":null,"affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Customers saw pull requests push event deliveries for Actions and Webhooks fail between 16:23 and 17:12 (UTC). We fixed the issue, and we are working to re-process push events for the affected time period.","created_at":"2023-11-28T17:36:08.163Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T17:36:08.163Z","display_at":"2023-11-28T17:36:08.163Z","deliver_notifications":true,"tweet_id":null,"id":"vqt2wp2lg24b","incident_id":"dsb3kn1zfdl7","custom_tweet":null,"affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Actions and Webhooks","created_at":"2023-11-28T17:24:53.385Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T17:24:53.385Z","display_at":"2023-11-28T17:24:53.385Z","deliver_notifications":true,"tweet_id":null,"id":"spx1976fp0gr","incident_id":"dsb3kn1zfdl7","custom_tweet":null,"affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-28T17:59:15.474Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-28T17:59:10.423Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Pull Requests, Issues, Webhooks and Actions","status":"resolved","created_at":"2023-11-27T19:43:00.029Z","updated_at":"2023-11-28T22:23:04.918Z","monitoring_at":null,"resolved_at":"2023-11-27T21:11:04.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/l7r357nldpzv","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-27T21:11:04.000Z","reminder_intervals":null,"id":"66vhjmd266r9","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"This incident has been resolved.\nOn November 27, 2023 at 18:46 UTC, we attempted to rotate our OpenID Connect (OIDC) authentication flow certificates. Due to an error in the certificate formatting, we uploaded an invalid certificate configuration that was not observed in our pre-production testing. Our background job servers were unable to start because a valid configuration is required at worker start up. As a result, users experienced delays in Pull Requests, Webhooks, Issues, Actions and Projects. Rollback of the change was slowed by the invalid certificate as our deployment system relied on the same certificate. Rollback was completed at 20:35 UTC. Most services recovered by 20:44 UTC. \n\u003cbr /\u003e\u003cbr /\u003e\nDelayed updates to Issues and Pull Requests were applied normally once the changes were rolled back. After the change was rolled back, a large queue of Actions-related jobs built up which included Pull Request, Pull Request review and Pull Request review comment events. About 2.3% of Actions jobs failed during the duration of the incident. Job queue times returned to normal once all remaining jobs were processed.\n\u003cbr /\u003e\u003cbr /\u003e\nWe are working to improve our certificate testing and rotation process to reduce the risk of customer-impacting errors.","created_at":"2023-11-27T21:11:04.116Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-28T22:23:04.915Z","display_at":"2023-11-27T21:11:04.000Z","deliver_notifications":true,"tweet_id":null,"id":"gw5mkvtc521b","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Webhooks is operating normally.","created_at":"2023-11-27T20:44:28.246Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:44:28.246Z","display_at":"2023-11-27T20:44:28.246Z","deliver_notifications":true,"tweet_id":null,"id":"p08m73k9yrq2","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Issues is operating normally.","created_at":"2023-11-27T20:44:01.611Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:44:01.611Z","display_at":"2023-11-27T20:44:01.611Z","deliver_notifications":true,"tweet_id":null,"id":"grj2c3wqxf7h","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pull Requests is operating normally.","created_at":"2023-11-27T20:43:49.074Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:43:49.074Z","display_at":"2023-11-27T20:43:49.074Z","deliver_notifications":true,"tweet_id":null,"id":"bkqz16ychvp9","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Actions customers are experiencing workflow start delays as part of the ongoing PRs incident. We are seeing previously delayed runs kick off and will continue to monitor.","created_at":"2023-11-27T20:39:49.381Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:39:49.381Z","display_at":"2023-11-27T20:39:49.381Z","deliver_notifications":true,"tweet_id":null,"id":"3hjfl8cxlp4l","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-27T20:30:57.800Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:30:57.800Z","display_at":"2023-11-27T20:30:57.800Z","deliver_notifications":true,"tweet_id":null,"id":"czrfqv38lbcm","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Customers are also experiencing delays in webhook delivery and issue updates. We are seeing recovery and are continuing to monitor.","created_at":"2023-11-27T20:22:50.763Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:22:50.763Z","display_at":"2023-11-27T20:22:50.763Z","deliver_notifications":true,"tweet_id":null,"id":"xxvqh19s382z","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Webhooks is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-27T20:16:49.887Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:16:49.887Z","display_at":"2023-11-27T20:16:49.887Z","deliver_notifications":true,"tweet_id":null,"id":"547x08m6603z","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-27T20:16:45.267Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T20:16:45.267Z","display_at":"2023-11-27T20:16:45.267Z","deliver_notifications":true,"tweet_id":null,"id":"zl18wd6tpn3r","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Customers are seeing delays in pushed commits appearing on pull requests. We are currently investigating.","created_at":"2023-11-27T19:46:21.658Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T19:46:21.658Z","display_at":"2023-11-27T19:46:21.658Z","deliver_notifications":true,"tweet_id":null,"id":"3z716nxxcnvv","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Pull Requests","created_at":"2023-11-27T19:43:00.091Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-27T19:43:00.091Z","display_at":"2023-11-27T19:43:00.091Z","deliver_notifications":true,"tweet_id":null,"id":"m0w27ffx0mpw","incident_id":"66vhjmd266r9","custom_tweet":null,"affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-28T17:59:15.474Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2023-11-27T20:44:01.585Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2023-11-27T20:43:49.050Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-28T17:59:10.423Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Actions","status":"resolved","created_at":"2023-11-21T10:11:13.361Z","updated_at":"2023-11-22T19:22:14.901Z","monitoring_at":null,"resolved_at":"2023-11-21T11:27:43.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/xdmhxqgg6jh0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-21T10:11:13.355Z","reminder_intervals":null,"id":"x39xrr5m11b3","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On November 21, 2023, at 09:50 UTC GitHub Actions jobs encountered delays due to an incident in our background job service caused by excessive rebalancing in a Kafka consumer group. After a quick mitigation, we began to see recovery on the job queues by 10:02 UTC. During this time window 100% of Actions jobs were delayed in starting for up to 11 minutes.\u003cbr /\u003e\u003cbr/\u003eUnfortunately, the rapid queue recovery sent a thundering herd of jobs to Actions hosted runner pools, causing a database deadlock that resulted in some hosted runner pools having increased latency when accepting new jobs. This affected only a small percentage of overall jobs, around 2%. Configuration changes led to a resolution and the system was fully recovered by 11:27 UTC and all in progress jobs were processed.\u003cbr/\u003e\u003cbr/\u003eThe incident is now resolved.","created_at":"2023-11-21T11:27:43.906Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-22T19:22:14.885Z","display_at":"2023-11-21T11:27:43.000Z","deliver_notifications":true,"tweet_id":null,"id":"y8vgyb26fm0c","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We've applied a mitigation to fix the issues with queuing and running Actions jobs. We are seeing improvements in telemetry and are monitoring for full recovery.","created_at":"2023-11-21T11:12:40.555Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-21T11:12:40.555Z","display_at":"2023-11-21T11:12:40.555Z","deliver_notifications":true,"tweet_id":null,"id":"50zjy8yfyh90","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We have recovery for the underlying issue but are waiting for Actions queues to catch up. We expect this to be completed in less than 1 hour(s).","created_at":"2023-11-21T10:24:17.469Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-21T10:24:17.469Z","display_at":"2023-11-21T10:24:17.469Z","deliver_notifications":true,"tweet_id":null,"id":"rd8vfx8c5k9g","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Actions","created_at":"2023-11-21T10:11:13.441Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-21T10:11:13.441Z","display_at":"2023-11-21T10:11:13.441Z","deliver_notifications":true,"tweet_id":null,"id":"024h3zsv6tg6","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-28T17:59:10.423Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"We are investigating reports of degraded performance.","status":"resolved","created_at":"2023-11-15T09:50:47.994Z","updated_at":"2023-11-18T02:17:20.825Z","monitoring_at":null,"resolved_at":"2023-11-15T11:34:14.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/t4g0f2hdnhp0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-15T11:34:14.000Z","reminder_intervals":null,"id":"mnv0g944fncw","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On 2023-11-15, from 09:44 to 10:42 UTC, some GitHub customers experienced increased latency or errors accessing repo data.\u003cbr /\u003e\u003cbr /\u003eHigh concurrent access to a specific git object exposed a bug that forced a backend service to perform excessive calculations, overloading the service. Access to this repo was paused while load was re-rerouted, mitigating the problem.\nThe conditions that triggered the expensive operations have been identified and refactored.","created_at":"2023-11-15T11:34:14.238Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-18T02:17:20.820Z","display_at":"2023-11-15T11:34:14.000Z","deliver_notifications":true,"tweet_id":null,"id":"wmq9z9jmjmrg","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Error rates and performance have returned to normal.","created_at":"2023-11-15T11:33:26.811Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T11:33:26.811Z","display_at":"2023-11-15T11:33:26.811Z","deliver_notifications":true,"tweet_id":null,"id":"w0r472xq2pf9","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We have identified the source of the issue and have removed the additional load from the service. Sporadic delays in pull request experiences and intermittent 500s are still occurring and impacting a very small percentage of traffic. Next update is expected within 30 minutes.","created_at":"2023-11-15T11:21:38.081Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T11:21:38.081Z","display_at":"2023-11-15T11:21:38.081Z","deliver_notifications":true,"tweet_id":null,"id":"08r6lgtjrqb0","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are seeing connectivity issues between some of our systems and git backend services. This is causing intermittent error responses and delays in pull request experiences for a very small percentage of traffic. We are investigating mitigations and expect to provide another update within 30 minutes.","created_at":"2023-11-15T11:04:10.342Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T11:04:10.342Z","display_at":"2023-11-15T11:04:10.342Z","deliver_notifications":false,"tweet_id":null,"id":"j1z1bd3v3tfl","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-11-15T09:50:48.096Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T09:50:48.096Z","display_at":"2023-11-15T09:50:48.096Z","deliver_notifications":true,"tweet_id":null,"id":"zft6pf0cqb5c","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"We are investigating reports of degraded performance.","status":"resolved","created_at":"2023-11-13T21:13:31.939Z","updated_at":"2023-11-16T17:51:42.694Z","monitoring_at":null,"resolved_at":"2023-11-13T21:38:40.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/768mh3pz82hk","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-13T21:13:31.927Z","reminder_intervals":null,"id":"knl5pxnx0byt","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Between 20:35 and 21:38 we experienced up to a 20 minute delay delivering around 30,000 notifications due to side effects of some planned maintenance on supporting systems. We have noted the unexpected user impact of this type of maintenance and will address it in future maintenance planning.","created_at":"2023-11-13T21:38:40.476Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-16T17:51:42.667Z","display_at":"2023-11-13T21:38:40.000Z","deliver_notifications":true,"tweet_id":null,"id":"t07zhwmt0vvr","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"An issue related to notifications has been resolved. Users should again be seeing their notifications.","created_at":"2023-11-13T21:38:23.207Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-13T21:38:23.207Z","display_at":"2023-11-13T21:38:23.207Z","deliver_notifications":true,"tweet_id":null,"id":"gpmqtqplt538","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We're seeing issues related to notifications.","created_at":"2023-11-13T21:15:31.689Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-13T21:15:31.689Z","display_at":"2023-11-13T21:15:31.689Z","deliver_notifications":true,"tweet_id":null,"id":"03wg4plbx5qx","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-11-13T21:13:32.008Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-13T21:13:32.008Z","display_at":"2023-11-13T21:13:32.008Z","deliver_notifications":true,"tweet_id":null,"id":"wb589hrwf42n","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Incident with Pages, Webhooks and Actions","status":"resolved","created_at":"2023-11-11T01:26:52.998Z","updated_at":"2023-11-16T21:41:15.672Z","monitoring_at":null,"resolved_at":"2023-11-11T02:14:08.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/t4bqfh6wb30t","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-11T01:26:52.990Z","reminder_intervals":null,"id":"wyk0ns67krlz","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On November 11, 2023, at 1:00 UTC, GitHub background jobs encountered delays lasting up to 50 minutes. This delay affected various services utilizing background jobs, including Actions, Webhooks, Pull Requests, and Pages. The impact persisted for approximately one hour until 2:10 UTC.\n\nDuring the incident, some customers experienced delays in starting Github Actions workflow runs and Pages builds. We estimate that about 10% of Actions workflow runs were delayed during the impact window and 99% of Pages builds failed from 1:00 UTC to 1:20 UTC. Users may have experienced a delay in seeing recent pushes reflected in pull request views. This delay averaged between 5 and 10 minutes and affected up to 30% of pull request page views during the incident. 1% of pull request page views experienced delays of up to 60 minutes. Finally, 30% of webhook deliveries in this window missed our target of being delivered within 1 minute of the triggering event.\n\nThis incident was caused by excessive rebalancing in our Kafka consumer group that feeds our background job system. We have altered our Kafka configuration to reduce the likelihood of this issue, created diagnostic tools to identify future causes, and will be breaking up this relay into multiple groups to limit the blast radius if the problem does reoccur.","created_at":"2023-11-11T02:14:08.550Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-16T21:41:15.657Z","display_at":"2023-11-11T02:14:08.000Z","deliver_notifications":true,"tweet_id":null,"id":"72znscq0llp7","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pages is operating normally.","created_at":"2023-11-11T02:14:00.569Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T02:14:00.569Z","display_at":"2023-11-11T02:14:00.569Z","deliver_notifications":true,"tweet_id":null,"id":"hkl0xyy7xs0m","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Actions is operating normally.","created_at":"2023-11-11T02:13:43.891Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T02:13:43.891Z","display_at":"2023-11-11T02:13:43.891Z","deliver_notifications":true,"tweet_id":null,"id":"jvy4kqymgd9m","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Rebalancing completed and job queues are improving.  We continue to monitor for full recovery of Webhooks, Actions, and Pages workflows.","created_at":"2023-11-11T01:53:33.689Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:53:33.689Z","display_at":"2023-11-11T01:53:33.689Z","deliver_notifications":true,"tweet_id":null,"id":"xcppqqjcg0zw","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-11T01:42:08.285Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:42:08.285Z","display_at":"2023-11-11T01:42:08.285Z","deliver_notifications":true,"tweet_id":null,"id":"zp4j86r3rlmx","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Webhooks is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-11T01:41:45.140Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:41:45.140Z","display_at":"2023-11-11T01:41:45.140Z","deliver_notifications":true,"tweet_id":null,"id":"jntybm9rj8nv","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Pages builds, webhooks, and other workflows were delayed starting at 1:00 UTC.  We have failed over the service that was contributing to the delays and see successful processing.  We are continuing to monitor for full recovery","created_at":"2023-11-11T01:40:11.531Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:40:11.531Z","display_at":"2023-11-11T01:40:11.531Z","deliver_notifications":true,"tweet_id":null,"id":"mws9cxy8fm1w","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Pages","created_at":"2023-11-11T01:26:53.081Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:26:53.081Z","display_at":"2023-11-11T01:26:53.081Z","deliver_notifications":true,"tweet_id":null,"id":"f4f364mp4b43","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-28T17:59:15.474Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-28T17:59:10.423Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2023-11-11T02:14:00.543Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with API Requests and Issues","status":"resolved","created_at":"2023-11-07T13:44:31.238Z","updated_at":"2023-11-16T17:38:36.513Z","monitoring_at":null,"resolved_at":"2023-11-07T14:25:40.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/8tyvmqmng1c0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-07T13:44:31.232Z","reminder_intervals":null,"id":"m61vxgn4kvh2","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Our internal search infrastructure experienced increased latency and timeouts between 13:15 and 14:05 UTC leading to some end user timeouts and slow responses for any requests that made use of that subsystem. This included but was not limited to: user search, repository search, releases, and audit logs.\n\nWe mitigated the issue by migrating traffic to an older version of our search clusters and are investigating what caused the performance issues in our new clusters.","created_at":"2023-11-07T14:25:40.875Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-16T17:38:36.499Z","display_at":"2023-11-07T14:25:40.000Z","deliver_notifications":true,"tweet_id":null,"id":"hc6vfn7bv2mv","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"API Requests is operating normally.","created_at":"2023-11-07T14:25:35.886Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T14:25:35.886Z","display_at":"2023-11-07T14:25:35.886Z","deliver_notifications":true,"tweet_id":null,"id":"wp9xs6h8bnzt","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Response times stabilized back to normal at 13:58 UTC.  We are continuing to monitor the slow dependency to ensure it's stable before resolving this incident.","created_at":"2023-11-07T14:15:29.896Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T14:15:29.896Z","display_at":"2023-11-07T14:15:29.896Z","deliver_notifications":true,"tweet_id":null,"id":"dkxlk3pnnnfh","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We're seeing intermittent spikes in latency of API requests and page loads.  We are investigating but do not have an ETA at this time.","created_at":"2023-11-07T14:03:35.646Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T14:03:35.646Z","display_at":"2023-11-07T14:03:35.646Z","deliver_notifications":true,"tweet_id":null,"id":"2yc60nvjj1y1","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of issues with service(s): Issues, API Requests. We will continue to keep users updated on progress towards mitigation.","created_at":"2023-11-07T13:50:26.616Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T13:50:26.616Z","display_at":"2023-11-07T13:50:26.616Z","deliver_notifications":true,"tweet_id":null,"id":"4qrdl67rgf45","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-07T13:46:01.377Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T13:46:01.377Z","display_at":"2023-11-07T13:46:01.377Z","deliver_notifications":true,"tweet_id":null,"id":"lsqdktqxx5lr","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for API Requests","created_at":"2023-11-07T13:44:31.331Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T13:44:31.331Z","display_at":"2023-11-07T13:44:31.331Z","deliver_notifications":true,"tweet_id":null,"id":"r01y5ylrrmz4","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2023-11-07T14:25:35.851Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2023-11-27T20:44:01.585Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Git Operations, Issues, Pull Requests, Actions, API Requests, Codespaces, Packages, Pages and Webhooks","status":"resolved","created_at":"2023-11-03T16:10:45.751Z","updated_at":"2023-11-03T20:32:02.341Z","monitoring_at":null,"resolved_at":"2023-11-03T19:21:48.000Z","impact":"major","shortlink":"https://web.archive.org/web/20231206141002/https://stspg.io/hz26zhwl0wxj","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-03T19:21:48.000Z","reminder_intervals":null,"id":"xb30mby9fs5x","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"A performance and resilience optimization to the authorization microservice contained a memory leak that was exposed under high traffic. This resulted in a number of pages returning 404’s that should not have. Testing the build in our canary ring did not expose the service to sufficient traffic to discover the leak, allowing it to graduate to production at 6:37 PM UTC.  The memory leak under high load caused pods to crash repeatedly starting at 6:42 PM UTC, failing authorization checks. These failures triggered alerts at 6:44 PM UTC. Rolling back the authorization service change was delayed as parts of the deployment infrastructure relied on the authorization service and required manual intervention to complete. Rollback completed at 7:08 PM UTC and all impacted GitHub features recovered after pods came back online.  We are evaluating changes to our rollout strategy to better detect this sooner and with less impact, changes to remove the dependency between authorization services and deployment rollback, and incident response improvements to reduce the overall time to recover.\n\u003cbr /\u003e\u003cbr /\u003e\nThis incident is unrelated to the Slack integration incident.  The combined status updates are a limitation of our status reporting tooling and we recognize the confusion this creates. Work to address this was already in progress and will be complete this month.","created_at":"2023-11-03T19:21:48.601Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T20:32:02.336Z","display_at":"2023-11-03T19:21:48.000Z","deliver_notifications":true,"tweet_id":null,"id":"jlg7lrf5l7nz","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Slack notifications have recovered.","created_at":"2023-11-03T19:17:18.900Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:17:18.900Z","display_at":"2023-11-03T19:17:18.900Z","deliver_notifications":true,"tweet_id":null,"id":"8yk1rjy9hzx1","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Webhooks is operating normally.","created_at":"2023-11-03T19:15:56.350Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:56.350Z","display_at":"2023-11-03T19:15:56.350Z","deliver_notifications":true,"tweet_id":null,"id":"p4k5zg3c7kf6","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pull Requests is operating normally.","created_at":"2023-11-03T19:15:51.345Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:51.345Z","display_at":"2023-11-03T19:15:51.345Z","deliver_notifications":true,"tweet_id":null,"id":"thhrx1ngxr99","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Issues is operating normally.","created_at":"2023-11-03T19:15:45.043Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:45.043Z","display_at":"2023-11-03T19:15:45.043Z","deliver_notifications":true,"tweet_id":null,"id":"xz0flqqytjtt","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Git Operations is operating normally.","created_at":"2023-11-03T19:15:40.393Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:40.393Z","display_at":"2023-11-03T19:15:40.393Z","deliver_notifications":true,"tweet_id":null,"id":"fz5hxmd70yq8","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"API Requests is operating normally.","created_at":"2023-11-03T19:15:36.451Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:36.451Z","display_at":"2023-11-03T19:15:36.451Z","deliver_notifications":true,"tweet_id":null,"id":"17jyrrftm1kr","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Actions is operating normally.","created_at":"2023-11-03T19:15:30.594Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:30.594Z","display_at":"2023-11-03T19:15:30.594Z","deliver_notifications":true,"tweet_id":null,"id":"pp9lpkmpd0mc","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pages is operating normally.","created_at":"2023-11-03T19:14:57.969Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:14:57.969Z","display_at":"2023-11-03T19:14:57.969Z","deliver_notifications":true,"tweet_id":null,"id":"l6lmc65ltdy9","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Codespaces is operating normally.","created_at":"2023-11-03T19:14:52.023Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:14:52.023Z","display_at":"2023-11-03T19:14:52.023Z","deliver_notifications":true,"tweet_id":null,"id":"yfhrjbcwzl9w","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Packages is operating normally.","created_at":"2023-11-03T19:13:42.750Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:13:42.750Z","display_at":"2023-11-03T19:13:42.750Z","deliver_notifications":true,"tweet_id":null,"id":"f97fyfvpbq0f","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"major_outage","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We have completed the rollback and are monitoring recovery.","created_at":"2023-11-03T19:10:25.058Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:10:25.058Z","display_at":"2023-11-03T19:10:25.058Z","deliver_notifications":true,"tweet_id":null,"id":"9vh167z1yww6","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"major_outage","new_status":"major_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We’re in the process of rolling back an authorization-related change that is causing 404s and other errors.","created_at":"2023-11-03T19:09:13.181Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:09:13.181Z","display_at":"2023-11-03T19:09:13.181Z","deliver_notifications":true,"tweet_id":null,"id":"nwlkcz6xwf6n","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"major_outage","new_status":"major_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Packages is experiencing degraded availability. We are continuing to investigate.","created_at":"2023-11-03T19:08:03.325Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:08:03.325Z","display_at":"2023-11-03T19:08:03.325Z","deliver_notifications":true,"tweet_id":null,"id":"40xgbvvstbqj","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"major_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Pages is experiencing degraded availability. We are continuing to investigate.","created_at":"2023-11-03T19:07:13.410Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:07:13.410Z","display_at":"2023-11-03T19:07:13.410Z","deliver_notifications":true,"tweet_id":null,"id":"w4gnxb9qlnzg","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded availability. We are continuing to investigate.","created_at":"2023-11-03T19:01:57.050Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:01:57.050Z","display_at":"2023-11-03T19:01:57.050Z","deliver_notifications":true,"tweet_id":null,"id":"gzmw33359j19","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Packages is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T19:00:08.579Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:00:08.579Z","display_at":"2023-11-03T19:00:08.579Z","deliver_notifications":true,"tweet_id":null,"id":"lpd1d91l5yrv","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Codespaces is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:59:59.590Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:59:59.590Z","display_at":"2023-11-03T18:59:59.590Z","deliver_notifications":true,"tweet_id":null,"id":"yc78jgcyycdz","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"API Requests is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:58:49.980Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:58:49.980Z","display_at":"2023-11-03T18:58:49.980Z","deliver_notifications":true,"tweet_id":null,"id":"j1j1y4j8b946","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:56:07.576Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:56:07.576Z","display_at":"2023-11-03T18:56:07.576Z","deliver_notifications":true,"tweet_id":null,"id":"t1n2fn7h4115","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Pull Requests is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:55:34.299Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:55:34.299Z","display_at":"2023-11-03T18:55:34.299Z","deliver_notifications":true,"tweet_id":null,"id":"7v0gsbft20lx","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:55:29.289Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:55:29.289Z","display_at":"2023-11-03T18:55:29.289Z","deliver_notifications":true,"tweet_id":null,"id":"54f46sf6tj96","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Git Operations is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:55:24.624Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:55:24.624Z","display_at":"2023-11-03T18:55:24.624Z","deliver_notifications":true,"tweet_id":null,"id":"5v3mn97n6g76","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"The delayed Slack notifications should be fully processed in about 30 minutes.","created_at":"2023-11-03T18:25:02.641Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:25:02.641Z","display_at":"2023-11-03T18:25:02.641Z","deliver_notifications":true,"tweet_id":null,"id":"kgglfj0vgnff","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Delayed Slack notifications are processing and the queue is expected to clear in just over an hour.","created_at":"2023-11-03T17:53:03.409Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T17:53:03.409Z","display_at":"2023-11-03T17:53:03.409Z","deliver_notifications":true,"tweet_id":null,"id":"2y6cj08dt24f","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Users may see delayed Slack notifications coming through as the queue is processed.","created_at":"2023-11-03T17:25:09.777Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T17:25:09.777Z","display_at":"2023-11-03T17:25:09.777Z","deliver_notifications":true,"tweet_id":null,"id":"shmzrxg4trbn","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Fix has been deployed, Slack integrations are recovering","created_at":"2023-11-03T17:22:53.469Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T17:22:53.469Z","display_at":"2023-11-03T17:22:53.469Z","deliver_notifications":true,"tweet_id":null,"id":"mjnxxs8v1dpw","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are testing a fix for Slack integrations.","created_at":"2023-11-03T16:56:29.069Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T16:56:29.069Z","display_at":"2023-11-03T16:56:29.069Z","deliver_notifications":true,"tweet_id":null,"id":"gw5qkf30dtmw","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are aware of issues with Slack integration and are working on resolving the problem.","created_at":"2023-11-03T16:11:57.725Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T16:11:57.725Z","display_at":"2023-11-03T16:11:57.725Z","deliver_notifications":true,"tweet_id":null,"id":"khyjgdbhff2v","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-11-03T16:10:45.818Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T16:10:45.818Z","display_at":"2023-11-03T16:10:45.818Z","deliver_notifications":true,"tweet_id":null,"id":"b594m834656z","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2023-11-03T19:15:40.357Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2023-11-07T14:25:35.851Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-28T17:59:15.474Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2023-11-27T20:44:01.585Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2023-11-27T20:43:49.050Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-28T17:59:10.423Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Packages","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2023-11-03T19:13:42.721Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":false,"start_date":null,"id":"st3j38cctv9l","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2023-11-11T02:14:00.543Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2023-11-03T19:14:51.990Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null}]}]
}