{
  "page": {
    "id": "kctbh9vrtdwd",
    "name": "GitHub",
    "url": "https://web.archive.org/web/20231126145616/https://www.githubstatus.com"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2023-11-03T19:15:40.357Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2023-11-07T14:25:35.851Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-11T02:14:08.520Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Visit www.githubstatus.com for more information","created_at":"2018-12-05T19:39:40.838Z","updated_at":"2022-09-07T00:08:33.519Z","position":4,"description":null,"showcase":false,"start_date":null,"id":"0l2p9nhqnxpd","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2023-11-07T14:25:40.847Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2023-11-03T19:15:51.320Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-21T11:27:43.877Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Packages","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2023-11-03T19:13:42.721Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":false,"start_date":null,"id":"st3j38cctv9l","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2023-11-11T02:14:00.543Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2023-11-03T19:14:51.990Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2023-10-25T22:15:54.589Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}],
  "incidents": [{"name":"Incident with Actions","status":"resolved","created_at":"2023-11-21T10:11:13.361Z","updated_at":"2023-11-22T19:22:14.901Z","monitoring_at":null,"resolved_at":"2023-11-21T11:27:43.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/xdmhxqgg6jh0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-21T10:11:13.355Z","reminder_intervals":null,"id":"x39xrr5m11b3","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On November 21, 2023, at 09:50 UTC GitHub Actions jobs encountered delays due to an incident in our background job service caused by excessive rebalancing in a Kafka consumer group. After a quick mitigation, we began to see recovery on the job queues by 10:02 UTC. During this time window 100% of Actions jobs were delayed in starting for up to 11 minutes.\u003cbr /\u003e\u003cbr/\u003eUnfortunately, the rapid queue recovery sent a thundering herd of jobs to Actions hosted runner pools, causing a database deadlock that resulted in some hosted runner pools having increased latency when accepting new jobs. This affected only a small percentage of overall jobs, around 2%. Configuration changes led to a resolution and the system was fully recovered by 11:27 UTC and all in progress jobs were processed.\u003cbr/\u003e\u003cbr/\u003eThe incident is now resolved.","created_at":"2023-11-21T11:27:43.906Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-22T19:22:14.885Z","display_at":"2023-11-21T11:27:43.000Z","deliver_notifications":true,"tweet_id":null,"id":"y8vgyb26fm0c","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We've applied a mitigation to fix the issues with queuing and running Actions jobs. We are seeing improvements in telemetry and are monitoring for full recovery.","created_at":"2023-11-21T11:12:40.555Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-21T11:12:40.555Z","display_at":"2023-11-21T11:12:40.555Z","deliver_notifications":true,"tweet_id":null,"id":"50zjy8yfyh90","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We have recovery for the underlying issue but are waiting for Actions queues to catch up. We expect this to be completed in less than 1 hour(s).","created_at":"2023-11-21T10:24:17.469Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-21T10:24:17.469Z","display_at":"2023-11-21T10:24:17.469Z","deliver_notifications":true,"tweet_id":null,"id":"rd8vfx8c5k9g","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Actions","created_at":"2023-11-21T10:11:13.441Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-21T10:11:13.441Z","display_at":"2023-11-21T10:11:13.441Z","deliver_notifications":true,"tweet_id":null,"id":"024h3zsv6tg6","incident_id":"x39xrr5m11b3","custom_tweet":null,"affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-21T11:27:43.877Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"We are investigating reports of degraded performance.","status":"resolved","created_at":"2023-11-15T09:50:47.994Z","updated_at":"2023-11-18T02:17:20.825Z","monitoring_at":null,"resolved_at":"2023-11-15T11:34:14.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/t4g0f2hdnhp0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-15T11:34:14.000Z","reminder_intervals":null,"id":"mnv0g944fncw","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On 2023-11-15, from 09:44 to 10:42 UTC, some GitHub customers experienced increased latency or errors accessing repo data.\u003cbr /\u003e\u003cbr /\u003eHigh concurrent access to a specific git object exposed a bug that forced a backend service to perform excessive calculations, overloading the service. Access to this repo was paused while load was re-rerouted, mitigating the problem.\nThe conditions that triggered the expensive operations have been identified and refactored.","created_at":"2023-11-15T11:34:14.238Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-18T02:17:20.820Z","display_at":"2023-11-15T11:34:14.000Z","deliver_notifications":true,"tweet_id":null,"id":"wmq9z9jmjmrg","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Error rates and performance have returned to normal.","created_at":"2023-11-15T11:33:26.811Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T11:33:26.811Z","display_at":"2023-11-15T11:33:26.811Z","deliver_notifications":true,"tweet_id":null,"id":"w0r472xq2pf9","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We have identified the source of the issue and have removed the additional load from the service. Sporadic delays in pull request experiences and intermittent 500s are still occurring and impacting a very small percentage of traffic. Next update is expected within 30 minutes.","created_at":"2023-11-15T11:21:38.081Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T11:21:38.081Z","display_at":"2023-11-15T11:21:38.081Z","deliver_notifications":true,"tweet_id":null,"id":"08r6lgtjrqb0","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are seeing connectivity issues between some of our systems and git backend services. This is causing intermittent error responses and delays in pull request experiences for a very small percentage of traffic. We are investigating mitigations and expect to provide another update within 30 minutes.","created_at":"2023-11-15T11:04:10.342Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T11:04:10.342Z","display_at":"2023-11-15T11:04:10.342Z","deliver_notifications":false,"tweet_id":null,"id":"j1z1bd3v3tfl","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-11-15T09:50:48.096Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-15T09:50:48.096Z","display_at":"2023-11-15T09:50:48.096Z","deliver_notifications":true,"tweet_id":null,"id":"zft6pf0cqb5c","incident_id":"mnv0g944fncw","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"We are investigating reports of degraded performance.","status":"resolved","created_at":"2023-11-13T21:13:31.939Z","updated_at":"2023-11-16T17:51:42.694Z","monitoring_at":null,"resolved_at":"2023-11-13T21:38:40.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/768mh3pz82hk","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-13T21:13:31.927Z","reminder_intervals":null,"id":"knl5pxnx0byt","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Between 20:35 and 21:38 we experienced up to a 20 minute delay delivering around 30,000 notifications due to side effects of some planned maintenance on supporting systems. We have noted the unexpected user impact of this type of maintenance and will address it in future maintenance planning.","created_at":"2023-11-13T21:38:40.476Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-16T17:51:42.667Z","display_at":"2023-11-13T21:38:40.000Z","deliver_notifications":true,"tweet_id":null,"id":"t07zhwmt0vvr","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"An issue related to notifications has been resolved. Users should again be seeing their notifications.","created_at":"2023-11-13T21:38:23.207Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-13T21:38:23.207Z","display_at":"2023-11-13T21:38:23.207Z","deliver_notifications":true,"tweet_id":null,"id":"gpmqtqplt538","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We're seeing issues related to notifications.","created_at":"2023-11-13T21:15:31.689Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-13T21:15:31.689Z","display_at":"2023-11-13T21:15:31.689Z","deliver_notifications":true,"tweet_id":null,"id":"03wg4plbx5qx","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-11-13T21:13:32.008Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-13T21:13:32.008Z","display_at":"2023-11-13T21:13:32.008Z","deliver_notifications":true,"tweet_id":null,"id":"wb589hrwf42n","incident_id":"knl5pxnx0byt","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Incident with Pages, Webhooks and Actions","status":"resolved","created_at":"2023-11-11T01:26:52.998Z","updated_at":"2023-11-16T21:41:15.672Z","monitoring_at":null,"resolved_at":"2023-11-11T02:14:08.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/t4bqfh6wb30t","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-11T01:26:52.990Z","reminder_intervals":null,"id":"wyk0ns67krlz","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On November 11, 2023, at 1:00 UTC, GitHub background jobs encountered delays lasting up to 50 minutes. This delay affected various services utilizing background jobs, including Actions, Webhooks, Pull Requests, and Pages. The impact persisted for approximately one hour until 2:10 UTC.\n\nDuring the incident, some customers experienced delays in starting Github Actions workflow runs and Pages builds. We estimate that about 10% of Actions workflow runs were delayed during the impact window and 99% of Pages builds failed from 1:00 UTC to 1:20 UTC. Users may have experienced a delay in seeing recent pushes reflected in pull request views. This delay averaged between 5 and 10 minutes and affected up to 30% of pull request page views during the incident. 1% of pull request page views experienced delays of up to 60 minutes. Finally, 30% of webhook deliveries in this window missed our target of being delivered within 1 minute of the triggering event.\n\nThis incident was caused by excessive rebalancing in our Kafka consumer group that feeds our background job system. We have altered our Kafka configuration to reduce the likelihood of this issue, created diagnostic tools to identify future causes, and will be breaking up this relay into multiple groups to limit the blast radius if the problem does reoccur.","created_at":"2023-11-11T02:14:08.550Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-16T21:41:15.657Z","display_at":"2023-11-11T02:14:08.000Z","deliver_notifications":true,"tweet_id":null,"id":"72znscq0llp7","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pages is operating normally.","created_at":"2023-11-11T02:14:00.569Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T02:14:00.569Z","display_at":"2023-11-11T02:14:00.569Z","deliver_notifications":true,"tweet_id":null,"id":"hkl0xyy7xs0m","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Actions is operating normally.","created_at":"2023-11-11T02:13:43.891Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T02:13:43.891Z","display_at":"2023-11-11T02:13:43.891Z","deliver_notifications":true,"tweet_id":null,"id":"jvy4kqymgd9m","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Rebalancing completed and job queues are improving.  We continue to monitor for full recovery of Webhooks, Actions, and Pages workflows.","created_at":"2023-11-11T01:53:33.689Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:53:33.689Z","display_at":"2023-11-11T01:53:33.689Z","deliver_notifications":true,"tweet_id":null,"id":"xcppqqjcg0zw","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-11T01:42:08.285Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:42:08.285Z","display_at":"2023-11-11T01:42:08.285Z","deliver_notifications":true,"tweet_id":null,"id":"zp4j86r3rlmx","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Webhooks is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-11T01:41:45.140Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:41:45.140Z","display_at":"2023-11-11T01:41:45.140Z","deliver_notifications":true,"tweet_id":null,"id":"jntybm9rj8nv","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Pages builds, webhooks, and other workflows were delayed starting at 1:00 UTC.  We have failed over the service that was contributing to the delays and see successful processing.  We are continuing to monitor for full recovery","created_at":"2023-11-11T01:40:11.531Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:40:11.531Z","display_at":"2023-11-11T01:40:11.531Z","deliver_notifications":true,"tweet_id":null,"id":"mws9cxy8fm1w","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Pages","created_at":"2023-11-11T01:26:53.081Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-11T01:26:53.081Z","display_at":"2023-11-11T01:26:53.081Z","deliver_notifications":true,"tweet_id":null,"id":"f4f364mp4b43","incident_id":"wyk0ns67krlz","custom_tweet":null,"affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-11T02:14:08.520Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-21T11:27:43.877Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2023-11-11T02:14:00.543Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with API Requests and Issues","status":"resolved","created_at":"2023-11-07T13:44:31.238Z","updated_at":"2023-11-16T17:38:36.513Z","monitoring_at":null,"resolved_at":"2023-11-07T14:25:40.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/8tyvmqmng1c0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-07T13:44:31.232Z","reminder_intervals":null,"id":"m61vxgn4kvh2","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Our internal search infrastructure experienced increased latency and timeouts between 13:15 and 14:05 UTC leading to some end user timeouts and slow responses for any requests that made use of that subsystem. This included but was not limited to: user search, repository search, releases, and audit logs.\n\nWe mitigated the issue by migrating traffic to an older version of our search clusters and are investigating what caused the performance issues in our new clusters.","created_at":"2023-11-07T14:25:40.875Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-16T17:38:36.499Z","display_at":"2023-11-07T14:25:40.000Z","deliver_notifications":true,"tweet_id":null,"id":"hc6vfn7bv2mv","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"API Requests is operating normally.","created_at":"2023-11-07T14:25:35.886Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T14:25:35.886Z","display_at":"2023-11-07T14:25:35.886Z","deliver_notifications":true,"tweet_id":null,"id":"wp9xs6h8bnzt","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Response times stabilized back to normal at 13:58 UTC.  We are continuing to monitor the slow dependency to ensure it's stable before resolving this incident.","created_at":"2023-11-07T14:15:29.896Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T14:15:29.896Z","display_at":"2023-11-07T14:15:29.896Z","deliver_notifications":true,"tweet_id":null,"id":"dkxlk3pnnnfh","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We're seeing intermittent spikes in latency of API requests and page loads.  We are investigating but do not have an ETA at this time.","created_at":"2023-11-07T14:03:35.646Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T14:03:35.646Z","display_at":"2023-11-07T14:03:35.646Z","deliver_notifications":true,"tweet_id":null,"id":"2yc60nvjj1y1","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of issues with service(s): Issues, API Requests. We will continue to keep users updated on progress towards mitigation.","created_at":"2023-11-07T13:50:26.616Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T13:50:26.616Z","display_at":"2023-11-07T13:50:26.616Z","deliver_notifications":true,"tweet_id":null,"id":"4qrdl67rgf45","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-07T13:46:01.377Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T13:46:01.377Z","display_at":"2023-11-07T13:46:01.377Z","deliver_notifications":true,"tweet_id":null,"id":"lsqdktqxx5lr","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for API Requests","created_at":"2023-11-07T13:44:31.331Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-07T13:44:31.331Z","display_at":"2023-11-07T13:44:31.331Z","deliver_notifications":true,"tweet_id":null,"id":"r01y5ylrrmz4","incident_id":"m61vxgn4kvh2","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2023-11-07T14:25:35.851Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2023-11-07T14:25:40.847Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Git Operations, Issues, Pull Requests, Actions, API Requests, Codespaces, Packages, Pages and Webhooks","status":"resolved","created_at":"2023-11-03T16:10:45.751Z","updated_at":"2023-11-03T20:32:02.341Z","monitoring_at":null,"resolved_at":"2023-11-03T19:21:48.000Z","impact":"major","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/hz26zhwl0wxj","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-11-03T19:21:48.000Z","reminder_intervals":null,"id":"xb30mby9fs5x","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"A performance and resilience optimization to the authorization microservice contained a memory leak that was exposed under high traffic. This resulted in a number of pages returning 404’s that should not have. Testing the build in our canary ring did not expose the service to sufficient traffic to discover the leak, allowing it to graduate to production at 6:37 PM UTC.  The memory leak under high load caused pods to crash repeatedly starting at 6:42 PM UTC, failing authorization checks. These failures triggered alerts at 6:44 PM UTC. Rolling back the authorization service change was delayed as parts of the deployment infrastructure relied on the authorization service and required manual intervention to complete. Rollback completed at 7:08 PM UTC and all impacted GitHub features recovered after pods came back online.  We are evaluating changes to our rollout strategy to better detect this sooner and with less impact, changes to remove the dependency between authorization services and deployment rollback, and incident response improvements to reduce the overall time to recover.\n\u003cbr /\u003e\u003cbr /\u003e\nThis incident is unrelated to the Slack integration incident.  The combined status updates are a limitation of our status reporting tooling and we recognize the confusion this creates. Work to address this was already in progress and will be complete this month.","created_at":"2023-11-03T19:21:48.601Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T20:32:02.336Z","display_at":"2023-11-03T19:21:48.000Z","deliver_notifications":true,"tweet_id":null,"id":"jlg7lrf5l7nz","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Slack notifications have recovered.","created_at":"2023-11-03T19:17:18.900Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:17:18.900Z","display_at":"2023-11-03T19:17:18.900Z","deliver_notifications":true,"tweet_id":null,"id":"8yk1rjy9hzx1","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Webhooks is operating normally.","created_at":"2023-11-03T19:15:56.350Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:56.350Z","display_at":"2023-11-03T19:15:56.350Z","deliver_notifications":true,"tweet_id":null,"id":"p4k5zg3c7kf6","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pull Requests is operating normally.","created_at":"2023-11-03T19:15:51.345Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:51.345Z","display_at":"2023-11-03T19:15:51.345Z","deliver_notifications":true,"tweet_id":null,"id":"thhrx1ngxr99","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Issues is operating normally.","created_at":"2023-11-03T19:15:45.043Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:45.043Z","display_at":"2023-11-03T19:15:45.043Z","deliver_notifications":true,"tweet_id":null,"id":"xz0flqqytjtt","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Git Operations is operating normally.","created_at":"2023-11-03T19:15:40.393Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:40.393Z","display_at":"2023-11-03T19:15:40.393Z","deliver_notifications":true,"tweet_id":null,"id":"fz5hxmd70yq8","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"API Requests is operating normally.","created_at":"2023-11-03T19:15:36.451Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:36.451Z","display_at":"2023-11-03T19:15:36.451Z","deliver_notifications":true,"tweet_id":null,"id":"17jyrrftm1kr","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Actions is operating normally.","created_at":"2023-11-03T19:15:30.594Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:15:30.594Z","display_at":"2023-11-03T19:15:30.594Z","deliver_notifications":true,"tweet_id":null,"id":"pp9lpkmpd0mc","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pages is operating normally.","created_at":"2023-11-03T19:14:57.969Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:14:57.969Z","display_at":"2023-11-03T19:14:57.969Z","deliver_notifications":true,"tweet_id":null,"id":"l6lmc65ltdy9","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Codespaces is operating normally.","created_at":"2023-11-03T19:14:52.023Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:14:52.023Z","display_at":"2023-11-03T19:14:52.023Z","deliver_notifications":true,"tweet_id":null,"id":"yfhrjbcwzl9w","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Packages is operating normally.","created_at":"2023-11-03T19:13:42.750Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:13:42.750Z","display_at":"2023-11-03T19:13:42.750Z","deliver_notifications":true,"tweet_id":null,"id":"f97fyfvpbq0f","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"major_outage","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We have completed the rollback and are monitoring recovery.","created_at":"2023-11-03T19:10:25.058Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:10:25.058Z","display_at":"2023-11-03T19:10:25.058Z","deliver_notifications":true,"tweet_id":null,"id":"9vh167z1yww6","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"major_outage","new_status":"major_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We’re in the process of rolling back an authorization-related change that is causing 404s and other errors.","created_at":"2023-11-03T19:09:13.181Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:09:13.181Z","display_at":"2023-11-03T19:09:13.181Z","deliver_notifications":true,"tweet_id":null,"id":"nwlkcz6xwf6n","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"major_outage","new_status":"major_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Packages is experiencing degraded availability. We are continuing to investigate.","created_at":"2023-11-03T19:08:03.325Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:08:03.325Z","display_at":"2023-11-03T19:08:03.325Z","deliver_notifications":true,"tweet_id":null,"id":"40xgbvvstbqj","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"major_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Pages is experiencing degraded availability. We are continuing to investigate.","created_at":"2023-11-03T19:07:13.410Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:07:13.410Z","display_at":"2023-11-03T19:07:13.410Z","deliver_notifications":true,"tweet_id":null,"id":"w4gnxb9qlnzg","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"major_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"major_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded availability. We are continuing to investigate.","created_at":"2023-11-03T19:01:57.050Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:01:57.050Z","display_at":"2023-11-03T19:01:57.050Z","deliver_notifications":true,"tweet_id":null,"id":"gzmw33359j19","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"major_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Packages is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T19:00:08.579Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T19:00:08.579Z","display_at":"2023-11-03T19:00:08.579Z","deliver_notifications":true,"tweet_id":null,"id":"lpd1d91l5yrv","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Codespaces is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:59:59.590Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:59:59.590Z","display_at":"2023-11-03T18:59:59.590Z","deliver_notifications":true,"tweet_id":null,"id":"yc78jgcyycdz","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"API Requests is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:58:49.980Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:58:49.980Z","display_at":"2023-11-03T18:58:49.980Z","deliver_notifications":true,"tweet_id":null,"id":"j1j1y4j8b946","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:56:07.576Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:56:07.576Z","display_at":"2023-11-03T18:56:07.576Z","deliver_notifications":true,"tweet_id":null,"id":"t1n2fn7h4115","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Pull Requests is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:55:34.299Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:55:34.299Z","display_at":"2023-11-03T18:55:34.299Z","deliver_notifications":true,"tweet_id":null,"id":"7v0gsbft20lx","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:55:29.289Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:55:29.289Z","display_at":"2023-11-03T18:55:29.289Z","deliver_notifications":true,"tweet_id":null,"id":"54f46sf6tj96","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Git Operations is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-11-03T18:55:24.624Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:55:24.624Z","display_at":"2023-11-03T18:55:24.624Z","deliver_notifications":true,"tweet_id":null,"id":"5v3mn97n6g76","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"The delayed Slack notifications should be fully processed in about 30 minutes.","created_at":"2023-11-03T18:25:02.641Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T18:25:02.641Z","display_at":"2023-11-03T18:25:02.641Z","deliver_notifications":true,"tweet_id":null,"id":"kgglfj0vgnff","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Delayed Slack notifications are processing and the queue is expected to clear in just over an hour.","created_at":"2023-11-03T17:53:03.409Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T17:53:03.409Z","display_at":"2023-11-03T17:53:03.409Z","deliver_notifications":true,"tweet_id":null,"id":"2y6cj08dt24f","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Users may see delayed Slack notifications coming through as the queue is processed.","created_at":"2023-11-03T17:25:09.777Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T17:25:09.777Z","display_at":"2023-11-03T17:25:09.777Z","deliver_notifications":true,"tweet_id":null,"id":"shmzrxg4trbn","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Fix has been deployed, Slack integrations are recovering","created_at":"2023-11-03T17:22:53.469Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T17:22:53.469Z","display_at":"2023-11-03T17:22:53.469Z","deliver_notifications":true,"tweet_id":null,"id":"mjnxxs8v1dpw","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are testing a fix for Slack integrations.","created_at":"2023-11-03T16:56:29.069Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T16:56:29.069Z","display_at":"2023-11-03T16:56:29.069Z","deliver_notifications":true,"tweet_id":null,"id":"gw5qkf30dtmw","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are aware of issues with Slack integration and are working on resolving the problem.","created_at":"2023-11-03T16:11:57.725Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T16:11:57.725Z","display_at":"2023-11-03T16:11:57.725Z","deliver_notifications":true,"tweet_id":null,"id":"khyjgdbhff2v","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2023-11-03T16:10:45.818Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-03T16:10:45.818Z","display_at":"2023-11-03T16:10:45.818Z","deliver_notifications":true,"tweet_id":null,"id":"b594m834656z","incident_id":"xb30mby9fs5x","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2023-11-03T19:15:40.357Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2023-11-07T14:25:35.851Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2023-11-11T02:14:08.520Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2023-11-07T14:25:40.847Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2023-11-03T19:15:51.320Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2023-11-21T11:27:43.877Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Packages","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2023-11-03T19:13:42.721Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":false,"start_date":null,"id":"st3j38cctv9l","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2023-11-11T02:14:00.543Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2023-11-03T19:14:51.990Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Copilot","status":"resolved","created_at":"2023-10-25T20:50:32.187Z","updated_at":"2023-11-01T16:32:33.283Z","monitoring_at":null,"resolved_at":"2023-10-25T22:15:54.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/t6g8trkmts9g","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-10-25T20:50:32.179Z","reminder_intervals":null,"id":"sbjdwn6mvht7","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Copilot completions are currently hosted in 4 regions globally: Central US, France, Switzerland and Japan. Users are typically routed to the nearest geographic region, but may be routed to other regions when the nearest region is unhealthy.\n\nBeginning at 2023-10-25 09:13 UTC, Copilot began experiencing outages of individual regions, lasting 12 minutes per region. These outages were due to the nodes hosting the completion model getting unhealthy due to a recent upgrade. There were intermittent outages in multiple regions with a subset of Copilot users experiencing completion errors. The outages were partial and varied across the different regions.\n\nIn order to prevent similar incidents from occurring in the future, we are focusing on improving our global load balancing of completion traffic during regional failures, in addition to determining and preventing the root cause of these outages.","created_at":"2023-10-25T22:15:54.610Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-01T16:32:33.268Z","display_at":"2023-10-25T22:15:54.000Z","deliver_notifications":true,"tweet_id":null,"id":"163db1ljtq22","incident_id":"sbjdwn6mvht7","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"The observed Copilot API error rate is around 5% of the requests. As a result, some of the Copilot code suggestions are skipped or not delivered on time.","created_at":"2023-10-25T21:37:05.818Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-25T21:37:05.818Z","display_at":"2023-10-25T21:37:05.818Z","deliver_notifications":true,"tweet_id":null,"id":"1fvdwmcbpv7t","incident_id":"sbjdwn6mvht7","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are seeing an impact in the US region as well. We continue the investigation.","created_at":"2023-10-25T21:19:19.530Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-25T21:19:19.530Z","display_at":"2023-10-25T21:19:19.530Z","deliver_notifications":true,"tweet_id":null,"id":"l3fpmzqcd3lz","incident_id":"sbjdwn6mvht7","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Copilot is experiencing intermittent issues in our Japan region. Engineers are currently investigating.","created_at":"2023-10-25T20:53:39.772Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-25T20:53:39.772Z","display_at":"2023-10-25T20:53:39.772Z","deliver_notifications":true,"tweet_id":null,"id":"rmh9xz3zsr29","incident_id":"sbjdwn6mvht7","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Copilot","created_at":"2023-10-25T20:50:32.266Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-25T20:50:32.266Z","display_at":"2023-10-25T20:50:32.266Z","deliver_notifications":true,"tweet_id":null,"id":"qg4zzqk17cv6","incident_id":"sbjdwn6mvht7","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2023-10-25T22:15:54.589Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Copilot","status":"resolved","created_at":"2023-10-25T12:10:13.595Z","updated_at":"2023-11-01T16:32:12.289Z","monitoring_at":null,"resolved_at":"2023-10-25T13:02:51.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/d1xnwpm2qkt0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-10-25T12:10:13.589Z","reminder_intervals":null,"id":"p351mbywbp0t","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Copilot completions are currently hosted in 4 regions globally: Central US, France, Switzerland and Japan. Users are typically routed to the nearest geographic region, but may be routed to other regions when the nearest region is unhealthy.\n\nBeginning at 2023-10-25 09:13 UTC, Copilot began experiencing outages of individual regions, lasting 12 minutes per region. These outages were due to the nodes hosting the completion model getting unhealthy due to a recent upgrade. There were intermittent outages in multiple regions with a subset of Copilot users experiencing completion errors. The outages were partial and varied across the different regions.\n\nIn order to prevent similar incidents from occurring in the future, we are focusing on improving our global load balancing of completion traffic during regional failures, in addition to determining and preventing the root cause of these outages.","created_at":"2023-10-25T13:02:51.296Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-11-01T16:32:12.268Z","display_at":"2023-10-25T13:02:51.000Z","deliver_notifications":true,"tweet_id":null,"id":"z8mct8dws98p","incident_id":"p351mbywbp0t","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We have applied a fix to help with Copilot performance. Initial signals show good recovery. We will continue to monitor for the time being and resolve when confident the issue has been resolved.","created_at":"2023-10-25T12:56:17.624Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-25T12:56:17.624Z","display_at":"2023-10-25T12:56:17.624Z","deliver_notifications":true,"tweet_id":null,"id":"00bdb7w7bqkb","incident_id":"p351mbywbp0t","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating degraded performance in Europe for Copilot. We will continue to keep users updated on progress towards mitigation.","created_at":"2023-10-25T12:29:04.509Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-25T12:29:04.509Z","display_at":"2023-10-25T12:29:04.509Z","deliver_notifications":true,"tweet_id":null,"id":"sh1dq2gqb128","incident_id":"p351mbywbp0t","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Copilot","created_at":"2023-10-25T12:10:13.658Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-25T12:10:13.658Z","display_at":"2023-10-25T12:10:13.658Z","deliver_notifications":true,"tweet_id":null,"id":"mfnpzq0y1047","incident_id":"p351mbywbp0t","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2023-10-25T22:15:54.589Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Git Operations","status":"resolved","created_at":"2023-10-22T15:16:00.777Z","updated_at":"2023-10-24T13:57:16.201Z","monitoring_at":null,"resolved_at":"2023-10-22T16:07:58.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/vb5qxf0s6gs0","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-10-22T15:16:00.768Z","reminder_intervals":null,"id":"lw4dvwltm025","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"This incident has been resolved.\n\nFrom 11:21 to 16:07 UTC some GitHub customers experienced errors cloning via workflows or via the command line.\n\nA third-party configuration change resulted in an unexpected behavior to our systems that resulted in Git clone failures. Once we detected the change we were able to disable it, and our systems started operating normally.\n\nWith the incident mitigated, we are working with our third-party provider to improve subsequent configuration change rollouts.","created_at":"2023-10-22T16:07:58.879Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-24T13:57:16.183Z","display_at":"2023-10-22T16:07:58.000Z","deliver_notifications":true,"tweet_id":null,"id":"7wdf5prdd6z6","incident_id":"lw4dvwltm025","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We have mitigated the cause of the issue and are awaiting positive confirmation from impacted customers that the issue is fully resolved.","created_at":"2023-10-22T15:58:41.506Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-22T15:58:41.506Z","display_at":"2023-10-22T15:58:41.506Z","deliver_notifications":true,"tweet_id":null,"id":"4wz47ktwsc9k","incident_id":"lw4dvwltm025","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are currently investigating reports from some customers encountering errors when cloning repositories via workflows or via the command line. We do not currently have an ETA for resolution. Next update in 30 minutes.","created_at":"2023-10-22T15:34:47.060Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-22T15:34:47.060Z","display_at":"2023-10-22T15:34:47.060Z","deliver_notifications":true,"tweet_id":null,"id":"yydr1y7cz8ww","incident_id":"lw4dvwltm025","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Git Operations","created_at":"2023-10-22T15:16:00.868Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-22T15:16:00.868Z","display_at":"2023-10-22T15:16:00.868Z","deliver_notifications":true,"tweet_id":null,"id":"1wwxct8bv6gh","incident_id":"lw4dvwltm025","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2023-11-03T19:15:40.357Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Codespaces","status":"resolved","created_at":"2023-10-17T11:14:24.786Z","updated_at":"2023-10-20T14:02:41.685Z","monitoring_at":null,"resolved_at":"2023-10-17T13:49:00.000Z","impact":"major","shortlink":"https://web.archive.org/web/20231126145616/https://stspg.io/w7qr4tvhhhkc","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2023-10-17T11:14:24.780Z","reminder_intervals":null,"id":"gkrfrz6r7flc","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"From 10:59 UTC to 13:48 UTC, GitHub Codespaces service was down or degraded due to an outage in our authentication service.This issue impacted 67% of users over this time period.\n\nOur service auth layer experienced throttling with our third party dependencies due to higher load. This impacted all user facing scenarios for Codespaces service. Our automated regional failover kicked in, but it failed to mitigate as this issue impacted the service globally. We mitigated manually by reducing load on external dependency.\n\nWith the incident mitigated, we are working to assess and implement scaling improvements to make our service more resilient with increasing load.","created_at":"2023-10-17T13:49:00.845Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-20T14:02:41.673Z","display_at":"2023-10-17T13:49:00.000Z","deliver_notifications":true,"tweet_id":null,"id":"gkml10l38d6f","incident_id":"gkrfrz6r7flc","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Codespaces is experiencing degraded performance. We are continuing to investigate.","created_at":"2023-10-17T13:46:42.182Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-17T13:46:42.182Z","display_at":"2023-10-17T13:46:42.182Z","deliver_notifications":true,"tweet_id":null,"id":"d0ggvr5g5ktl","incident_id":"gkrfrz6r7flc","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"major_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are continuing with efforts to mitigate Codespaces issues   and are beginning to see some Codespace creations succeed.","created_at":"2023-10-17T13:24:19.279Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-17T13:24:19.279Z","display_at":"2023-10-17T13:24:19.279Z","deliver_notifications":true,"tweet_id":null,"id":"zb30c2015kgl","incident_id":"gkrfrz6r7flc","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"major_outage","new_status":"major_outage"}]},{"status":"investigating","body":"We have identified an issue impacting most Codespaces operations and are working on a mitigation.","created_at":"2023-10-17T12:18:57.977Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-17T12:18:57.977Z","display_at":"2023-10-17T12:18:57.977Z","deliver_notifications":true,"tweet_id":null,"id":"htkv1x0qvzms","incident_id":"gkrfrz6r7flc","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"major_outage","new_status":"major_outage"}]},{"status":"investigating","body":"Codespaces is experiencing degraded availability. We are continuing to investigate.","created_at":"2023-10-17T11:18:38.412Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-17T11:18:38.412Z","display_at":"2023-10-17T11:18:38.412Z","deliver_notifications":true,"tweet_id":null,"id":"gttc4lbl6r6p","incident_id":"gkrfrz6r7flc","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"major_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Codespaces","created_at":"2023-10-17T11:14:24.845Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2023-10-17T11:14:24.845Z","display_at":"2023-10-17T11:14:24.845Z","deliver_notifications":true,"tweet_id":null,"id":"pzqqqkn8dt3v","incident_id":"gkrfrz6r7flc","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2023-11-03T19:14:51.990Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null}]}]
}