gossip: remove frequent gossiping of gossip client connections

a-robinson · irfansharif · commit 2114678be6d1 · 2022-12-19T17:43:20.000-05:00
These gossip-clients keys make up two thirds or more of the gossip
traffic in various large clusters I've inspected, consuming almost an
entire CPU core in the worst case I've seen. They don't provide enough
value to justify that sort of ongoing cost, so this commit removes them
entirely as well as the periodic logging of the gossip network and the
crdb_internal.gossip_network table, both of which relied on them.

Release note (backward-incompatible change): We've stopped
supporting/populating the crdb_internal.gossip_network table. It was an
internal table with no API guarantees (so perhaps no meriting a release
note?).

Release note (performance improvement): Significantly reduced CPU usage
of the underlying gossip network in large clusters.
diff --git a/pkg/cli/debug.go b/pkg/cli/debug.go
@@ -1069,8 +1069,6 @@ func parseGossipValues(gossipInfo *gossip.InfoStatus) (string, error) {
 				return "", errors.Wrapf(err, "failed to parse value for key %q", key)
 			}
 			output = append(output, fmt.Sprintf("%q: %+v", key, drainingInfo))
-		} else if strings.HasPrefix(key, gossip.KeyGossipClientsPrefix) {
-			output = append(output, fmt.Sprintf("%q: %v", key, string(bytes)))
 		}
 	}
 
diff --git a/pkg/cli/testdata/zip/partial1 b/pkg/cli/testdata/zip/partial1
@@ -78,7 +78,6 @@ debug zip --concurrency=1 --cpu-profile-duration=0s /dev/null
 [node 1] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/1/crdb_internal.feature_usage.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/1/crdb_internal.gossip_alerts.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/1/crdb_internal.gossip_liveness.txt... done
-[node 1] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/1/crdb_internal.gossip_network.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/1/crdb_internal.gossip_nodes.txt... done
 [node 1] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/1/crdb_internal.leases.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/1/crdb_internal.node_build_info.txt... done
@@ -179,9 +178,6 @@ debug zip --concurrency=1 --cpu-profile-duration=0s /dev/null
 [node 2] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/2/crdb_internal.gossip_liveness.txt...
 [node 2] retrieving SQL data for crdb_internal.gossip_liveness: last request failed: dial tcp ...
 [node 2] retrieving SQL data for crdb_internal.gossip_liveness: creating error output: debug/nodes/2/crdb_internal.gossip_liveness.txt.err.txt... done
-[node 2] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/2/crdb_internal.gossip_network.txt...
-[node 2] retrieving SQL data for crdb_internal.gossip_network: last request failed: dial tcp ...
-[node 2] retrieving SQL data for crdb_internal.gossip_network: creating error output: debug/nodes/2/crdb_internal.gossip_network.txt.err.txt... done
 [node 2] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/2/crdb_internal.gossip_nodes.txt...
 [node 2] retrieving SQL data for crdb_internal.gossip_nodes: last request failed: dial tcp ...
 [node 2] retrieving SQL data for crdb_internal.gossip_nodes: creating error output: debug/nodes/2/crdb_internal.gossip_nodes.txt.err.txt... done
@@ -263,7 +259,6 @@ debug zip --concurrency=1 --cpu-profile-duration=0s /dev/null
 [node 3] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/3/crdb_internal.feature_usage.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/3/crdb_internal.gossip_alerts.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/3/crdb_internal.gossip_liveness.txt... done
-[node 3] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/3/crdb_internal.gossip_network.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/3/crdb_internal.gossip_nodes.txt... done
 [node 3] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/3/crdb_internal.leases.txt... done
 [node 3] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/3/crdb_internal.node_build_info.txt... done
diff --git a/pkg/cli/testdata/zip/partial1_excluded b/pkg/cli/testdata/zip/partial1_excluded
@@ -78,7 +78,6 @@ debug zip /dev/null --concurrency=1 --exclude-nodes=2 --cpu-profile-duration=0
 [node 1] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/1/crdb_internal.feature_usage.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/1/crdb_internal.gossip_alerts.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/1/crdb_internal.gossip_liveness.txt... done
-[node 1] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/1/crdb_internal.gossip_network.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/1/crdb_internal.gossip_nodes.txt... done
 [node 1] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/1/crdb_internal.leases.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/1/crdb_internal.node_build_info.txt... done
@@ -172,7 +171,6 @@ debug zip /dev/null --concurrency=1 --exclude-nodes=2 --cpu-profile-duration=0
 [node 3] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/3/crdb_internal.feature_usage.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/3/crdb_internal.gossip_alerts.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/3/crdb_internal.gossip_liveness.txt... done
-[node 3] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/3/crdb_internal.gossip_network.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/3/crdb_internal.gossip_nodes.txt... done
 [node 3] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/3/crdb_internal.leases.txt... done
 [node 3] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/3/crdb_internal.node_build_info.txt... done
diff --git a/pkg/cli/testdata/zip/partial2 b/pkg/cli/testdata/zip/partial2
@@ -78,7 +78,6 @@ debug zip --concurrency=1 --cpu-profile-duration=0 /dev/null
 [node 1] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/1/crdb_internal.feature_usage.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/1/crdb_internal.gossip_alerts.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/1/crdb_internal.gossip_liveness.txt... done
-[node 1] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/1/crdb_internal.gossip_network.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/1/crdb_internal.gossip_nodes.txt... done
 [node 1] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/1/crdb_internal.leases.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/1/crdb_internal.node_build_info.txt... done
@@ -171,7 +170,6 @@ debug zip --concurrency=1 --cpu-profile-duration=0 /dev/null
 [node 3] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/3/crdb_internal.feature_usage.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/3/crdb_internal.gossip_alerts.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/3/crdb_internal.gossip_liveness.txt... done
-[node 3] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/3/crdb_internal.gossip_network.txt... done
 [node 3] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/3/crdb_internal.gossip_nodes.txt... done
 [node 3] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/3/crdb_internal.leases.txt... done
 [node 3] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/3/crdb_internal.node_build_info.txt... done
diff --git a/pkg/cli/testdata/zip/testzip b/pkg/cli/testdata/zip/testzip
@@ -81,7 +81,6 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/1/crdb_internal.feature_usage.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/1/crdb_internal.gossip_alerts.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/1/crdb_internal.gossip_liveness.txt... done
-[node 1] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/1/crdb_internal.gossip_network.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/1/crdb_internal.gossip_nodes.txt... done
 [node 1] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/1/crdb_internal.leases.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/1/crdb_internal.node_build_info.txt... done
diff --git a/pkg/cli/testdata/zip/testzip_concurrent b/pkg/cli/testdata/zip/testzip_concurrent
@@ -292,9 +292,6 @@ zip
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness...
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness: done
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness: writing output: debug/nodes/1/crdb_internal.gossip_liveness.txt...
-[node 1] retrieving SQL data for crdb_internal.gossip_network...
-[node 1] retrieving SQL data for crdb_internal.gossip_network: done
-[node 1] retrieving SQL data for crdb_internal.gossip_network: writing output: debug/nodes/1/crdb_internal.gossip_network.txt...
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes...
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes: done
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes: writing output: debug/nodes/1/crdb_internal.gossip_nodes.txt...
@@ -622,9 +619,6 @@ zip
 [node 2] retrieving SQL data for crdb_internal.gossip_liveness...
 [node 2] retrieving SQL data for crdb_internal.gossip_liveness: done
 [node 2] retrieving SQL data for crdb_internal.gossip_liveness: writing output: debug/nodes/2/crdb_internal.gossip_liveness.txt...
-[node 2] retrieving SQL data for crdb_internal.gossip_network...
-[node 2] retrieving SQL data for crdb_internal.gossip_network: done
-[node 2] retrieving SQL data for crdb_internal.gossip_network: writing output: debug/nodes/2/crdb_internal.gossip_network.txt...
 [node 2] retrieving SQL data for crdb_internal.gossip_nodes...
 [node 2] retrieving SQL data for crdb_internal.gossip_nodes: done
 [node 2] retrieving SQL data for crdb_internal.gossip_nodes: writing output: debug/nodes/2/crdb_internal.gossip_nodes.txt...
@@ -952,9 +946,6 @@ zip
 [node 3] retrieving SQL data for crdb_internal.gossip_liveness...
 [node 3] retrieving SQL data for crdb_internal.gossip_liveness: done
 [node 3] retrieving SQL data for crdb_internal.gossip_liveness: writing output: debug/nodes/3/crdb_internal.gossip_liveness.txt...
-[node 3] retrieving SQL data for crdb_internal.gossip_network...
-[node 3] retrieving SQL data for crdb_internal.gossip_network: done
-[node 3] retrieving SQL data for crdb_internal.gossip_network: writing output: debug/nodes/3/crdb_internal.gossip_network.txt...
 [node 3] retrieving SQL data for crdb_internal.gossip_nodes...
 [node 3] retrieving SQL data for crdb_internal.gossip_nodes: done
 [node 3] retrieving SQL data for crdb_internal.gossip_nodes: writing output: debug/nodes/3/crdb_internal.gossip_nodes.txt...
diff --git a/pkg/cli/testdata/zip/testzip_tenant b/pkg/cli/testdata/zip/testzip_tenant
@@ -110,9 +110,6 @@ debug zip --concurrency=1 --cpu-profile-duration=1s /dev/null
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/1/crdb_internal.gossip_liveness.txt...
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness: last request failed: ERROR: unimplemented: operation is unsupported in multi-tenancy mode (SQLSTATE 0A000)
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness: creating error output: debug/nodes/1/crdb_internal.gossip_liveness.txt.err.txt... done
-[node 1] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/1/crdb_internal.gossip_network.txt...
-[node 1] retrieving SQL data for crdb_internal.gossip_network: last request failed: ERROR: unimplemented: operation is unsupported in multi-tenancy mode (SQLSTATE 0A000)
-[node 1] retrieving SQL data for crdb_internal.gossip_network: creating error output: debug/nodes/1/crdb_internal.gossip_network.txt.err.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/1/crdb_internal.gossip_nodes.txt...
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes: last request failed: ERROR: unimplemented: operation is unsupported in multi-tenancy mode (SQLSTATE 0A000)
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes: creating error output: debug/nodes/1/crdb_internal.gossip_nodes.txt.err.txt... done
diff --git a/pkg/cli/testdata/zip/unavailable b/pkg/cli/testdata/zip/unavailable
@@ -90,7 +90,6 @@ debug zip --concurrency=1 --cpu-profile-duration=0 /dev/null --timeout=.5s
 [node 1] retrieving SQL data for crdb_internal.feature_usage... writing output: debug/nodes/1/crdb_internal.feature_usage.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_alerts... writing output: debug/nodes/1/crdb_internal.gossip_alerts.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_liveness... writing output: debug/nodes/1/crdb_internal.gossip_liveness.txt... done
-[node 1] retrieving SQL data for crdb_internal.gossip_network... writing output: debug/nodes/1/crdb_internal.gossip_network.txt... done
 [node 1] retrieving SQL data for crdb_internal.gossip_nodes... writing output: debug/nodes/1/crdb_internal.gossip_nodes.txt... done
 [node 1] retrieving SQL data for crdb_internal.leases... writing output: debug/nodes/1/crdb_internal.leases.txt... done
 [node 1] retrieving SQL data for crdb_internal.node_build_info... writing output: debug/nodes/1/crdb_internal.node_build_info.txt... done
diff --git a/pkg/cli/zip_table_registry.go b/pkg/cli/zip_table_registry.go
@@ -533,12 +533,6 @@ var zipInternalTablesPerNode = DebugZipTableRegistry{
 			"updated_at",
 		},
 	},
-	"crdb_internal.gossip_network": {
-		nonSensitiveCols: NonSensitiveColumns{
-			"source_id",
-			"target_id",
-		},
-	},
 	"crdb_internal.gossip_nodes": {
 		// `cluster_name` is hashed as we only care to see whether values are
 		// identical across nodes.
diff --git a/pkg/cli/zip_test.go b/pkg/cli/zip_test.go
@@ -76,6 +76,7 @@ table_name NOT IN (
 	'cross_db_references',
 	'databases',
 	'forward_dependencies',
+	'gossip_network',
 	'index_columns',
 	'kv_catalog_comments',
 	'kv_catalog_descriptor',
diff --git a/pkg/cmd/roachtest/tests/cli.go b/pkg/cmd/roachtest/tests/cli.go
@@ -109,7 +109,7 @@ func runCLINodeStatus(ctx context.Context, t test.Test, c cluster.Cluster) {
 	c.Stop(ctx, t.L(), option.DefaultStopOpts(), c.Node(3))
 	waitUntil([]string{
 		"is_available is_live",
-		"false true",
+		"false false",
 		"false false",
 		"false false",
 	})
diff --git a/pkg/cmd/roachtest/tests/gossip.go b/pkg/cmd/roachtest/tests/gossip.go
@@ -21,7 +21,6 @@ import (
 	"strconv"
 	"strings"
 	"time"
-	"unicode"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
@@ -47,52 +46,36 @@ func registerGossip(r registry.Registry) {
 		err := WaitFor3XReplication(ctx, t, c.Conn(ctx, t.L(), 1))
 		require.NoError(t, err)
 
-		// TODO(irfansharif): We could also look at gossip_liveness to determine
-		// cluster membership as seen by each gossip module, and ensure each
-		// node's gossip excludes the dead node and includes all other live
-		// ones.
-		gossipNetworkAccordingTo := func(node int) (network string) {
+		gossipNetworkAccordingTo := func(node int) (nodes []int) {
+			// Expiration timestamp format: <seconds>.<nanos>,<logical>. We'll
+			// capture just the <seconds> portion.
 			const query = `
-SELECT string_agg(source_id::TEXT || ':' || target_id::TEXT, ',')
-  FROM (SELECT * FROM crdb_internal.gossip_network ORDER BY source_id, target_id)
+SELECT node_id
+  FROM (SELECT node_id, to_timestamp(split_part(split_part(expiration, ',', 1), '.', 1)::FLOAT8) AS expiration
+		  FROM crdb_internal.gossip_liveness)
+ WHERE expiration > now();
 `
 
 			db := c.Conn(ctx, t.L(), node)
 			defer db.Close()
-			var s gosql.NullString
-			if err = db.QueryRow(query).Scan(&s); err != nil {
-				t.Fatal(err)
-			}
-			if s.Valid {
-				return s.String
-			}
-			return ""
-		}
 
-		nodesInNetworkAccordingTo := func(node int) (nodes []int, network string) {
-			split := func(c rune) bool {
-				return !unicode.IsNumber(c)
+			rows, err := db.Query(query)
+			if err != nil {
+				t.Fatal(err)
 			}
 
-			uniqueNodes := make(map[int]struct{})
-			network = gossipNetworkAccordingTo(node)
-			for _, idStr := range strings.FieldsFunc(network, split) {
-				nodeID, err := strconv.Atoi(idStr)
-				if err != nil {
-					t.Fatal(err)
-				}
-				uniqueNodes[nodeID] = struct{}{}
-			}
-			for node := range uniqueNodes {
-				nodes = append(nodes, node)
+			for rows.Next() {
+				var nodeID int
+				require.NoError(t, rows.Scan(&nodeID))
+				require.NotZero(t, nodeID)
+				nodes = append(nodes, nodeID)
 			}
 			sort.Ints(nodes)
-			return nodes, network
+			return nodes
 		}
 
 		gossipOK := func(start time.Time, deadNode int) bool {
 			var expLiveNodes []int
-			var expGossipNetwork string
 
 			for i := 1; i <= c.Spec().NodeCount; i++ {
 				if elapsed := timeutil.Since(start); elapsed >= 20*time.Second {
@@ -104,36 +87,35 @@ SELECT string_agg(source_id::TEXT || ':' || target_id::TEXT, ',')
 				}
 
 				t.L().Printf("%d: checking gossip\n", i)
-				liveNodes, gossipNetwork := nodesInNetworkAccordingTo(i)
+				liveNodes := gossipNetworkAccordingTo(i)
 				for _, id := range liveNodes {
 					if id == deadNode {
-						t.L().Printf("%d: gossip not ok (dead node %d present): %s (%.0fs)\n",
-							i, deadNode, gossipNetwork, timeutil.Since(start).Seconds())
+						t.L().Printf("%d: gossip not ok (dead node %d present) (%.0fs)\n",
+							i, deadNode, timeutil.Since(start).Seconds())
 						return false
 					}
 				}
 
 				if len(expLiveNodes) == 0 {
 					expLiveNodes = liveNodes
-					expGossipNetwork = gossipNetwork
 					continue
 				}
 
 				if len(liveNodes) != len(expLiveNodes) {
-					t.L().Printf("%d: gossip not ok (mismatched size of network: %s); expected %d, got %d (%.0fs)\n",
-						i, gossipNetwork, len(expLiveNodes), len(liveNodes), timeutil.Since(start).Seconds())
+					t.L().Printf("%d: gossip not ok (mismatched size of network); expected %d, got %d (%.0fs)\n",
+						i, len(expLiveNodes), len(liveNodes), timeutil.Since(start).Seconds())
 					return false
 				}
 
 				for i := range liveNodes {
 					if liveNodes[i] != expLiveNodes[i] {
 						t.L().Printf("%d: gossip not ok (mismatched view of live nodes); expected %s, got %s (%.0fs)\n",
-							i, gossipNetwork, expLiveNodes, liveNodes, timeutil.Since(start).Seconds())
+							i, expLiveNodes, liveNodes, timeutil.Since(start).Seconds())
 						return false
 					}
 				}
 			}
-			t.L().Printf("gossip ok: %s (size: %d) (%0.0fs)\n", expGossipNetwork, len(expLiveNodes), timeutil.Since(start).Seconds())
+			t.L().Printf("gossip ok (size: %d) (%0.0fs)\n", len(expLiveNodes), timeutil.Since(start).Seconds())
 			return true
 		}
 
diff --git a/pkg/gossip/client.go b/pkg/gossip/client.go
@@ -333,7 +333,6 @@ func (c *client) gossip(
 				}
 				if peerID == 0 && c.peerID != 0 {
 					peerID = c.peerID
-					g.updateClients()
 				}
 			}
 		}()
diff --git a/pkg/gossip/gossip.go b/pkg/gossip/gossip.go
diff --git a/pkg/gossip/gossip_test.go b/pkg/gossip/gossip_test.go
diff --git a/pkg/gossip/keys.go b/pkg/gossip/keys.go
diff --git a/pkg/sql/crdb_internal.go b/pkg/sql/crdb_internal.go

Original file line number	Diff line number	Diff line change
`@@ -1069,8 +1069,6 @@ func parseGossipValues(gossipInfo *gossip.InfoStatus) (string, error) {`
`1069`	`1069`	`return "", errors.Wrapf(err, "failed to parse value for key %q", key)`
`1070`	`1070`	`}`
`1071`	`1071`	`output = append(output, fmt.Sprintf("%q: %+v", key, drainingInfo))`
`1072`		`- } else if strings.HasPrefix(key, gossip.KeyGossipClientsPrefix) {`
`1073`		`- output = append(output, fmt.Sprintf("%q: %v", key, string(bytes)))`
`1074`	`1072`	`}`
`1075`	`1073`	`}`
`1076`	`1074`