kvserver: update distribution cluster settings to be system only

**Is your feature request related to a problem? Please describe.**
There are multiple  kv distribution cluster settings that relate to allocation and splitting which are `TenantWritable`. These settings, if set incorrectly, can quickly destabilize the cluster and lead to under-replication in extreme cases.

**Describe the solution you'd like**
Update the following settings to be `SystemOnly`

<details>
<summary>Settings</summary>

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/allocator/storepool/store_pool.go#L49-L51

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/allocator/storepool/store_pool.go#L61-L62

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/allocator/storepool/store_pool.go#L84-L86

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/replicastats/replica_stats.go#L40-L42

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/kvserverbase/base.go#L205-L208

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/replica_split_load.go#L31-L33

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/replica_split_load.go#L39-L41

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/replica_split_load.go#L57-L59

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/queue.go#L55-L56

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/store.go#L172-L175

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/store.go#L185-L187

https://github.com/cockroachdb/cockroach/blob/0495342e81d473901278993f6162b1d78b685bc1/pkg/kv/kvserver/store.go#L194-L196

</details>


<details>
<summary>All kvserver TenantWritable settings</summary>

```
❯ rg -A 1 TenantWritable
replica_write.go
46:     settings.TenantWritable,
47-     "kv.migration.migrate_application.timeout",

replica_follower_read.go
31:     settings.TenantWritable,
32-     "kv.closed_timestamp.follower_reads_enabled",

protectedts/settings.go
25:     settings.TenantWritable,
26-     "kv.protectedts.max_bytes",
--
35:     settings.TenantWritable,
36-     "kv.protectedts.max_spans",
--
45:     settings.TenantWritable,
46-     "kv.protectedts.poll_interval",

gc/gc.go
70:     settings.TenantWritable,
71-     "kv.gc.intent_age_threshold",
--
109:    settings.TenantWritable,
110-    "kv.gc.intent_cleanup_batch_size",
--
128:    settings.TenantWritable,
129-    "kv.gc.intent_cleanup_batch_byte_size",

reports/reporter.go
49:     settings.TenantWritable,
50-     "kv.replication_reports.interval",

kvserverbase/syncing_write.go
68:     settings.TenantWritable,
69-     "kv.bulk_sst.sync_size",

kvserverbase/base.go
207:    settings.TenantWritable,
208-    "kv.range_split.by_load_merge_delay",
--
232:    settings.TenantWritable,
233-    "kv.raft.command.max_size",

raft_transport.go
62:     settings.TenantWritable,
63-     "kv.raft.command.target_batch_size",

allocator/storepool/store_pool.go
50:     settings.TenantWritable,
51-     "server.failed_reservation_timeout",
--
62:     settings.TenantWritable,
63-     timeAfterStoreSuspectSettingName,
--
86:             settings.TenantWritable,
87-             timeUntilStoreDeadSettingName,

replica_split_load.go
32:     settings.TenantWritable,
33-     "kv.range_split.by_load_enabled",
--
40:     settings.TenantWritable,
41-     "kv.range_split.load_qps_threshold",
--
58:     settings.TenantWritable,
59-     "kv.range_split.load_cpu_threshold",

replica_backpressure.go
31:     settings.TenantWritable,
32-     "kv.range.backpressure_range_size_multiplier",
--
69:     settings.TenantWritable,
70-     "kv.range.backpressure_byte_tolerance",

batcheval/cmd_export.go
38:     settings.TenantWritable,
39-     SSTTargetSizeSetting,
--
56:     settings.TenantWritable,
57-     MaxExportOverageSetting,

batcheval/cmd_query_resolved_timestamp.go
33:     settings.TenantWritable,
34-     "kv.query_resolved_timestamp.intent_cleanup_age",

concurrency/concurrency_manager.go
63:     settings.TenantWritable,
64-     "kv.lock_table.maximum_lock_wait_queue_length",
--
96:     settings.TenantWritable,
97-     "kv.lock_table.discovered_locks_threshold_for_consulting_finalized_txn_cache",

concurrency/lock_table_waiter.go
42:     settings.TenantWritable,
43-     "kv.lock_table.coordinator_liveness_push_delay",
--
74:     settings.TenantWritable,
75-     "kv.lock_table.deadlock_detection_push_delay",

replicastats/replica_stats.go
41:     settings.TenantWritable,
42-     "kv.replica_stats.addsst_request_size_factor",

replica.go
94:     settings.TenantWritable,
95-     "kv.gc_ttl.strict_enforcement.enabled",

logstore/logstore.go
41:     settings.TenantWritable,
42-     "kv.raft_log.disable_synchronization_unsafe",
--
50:     settings.TenantWritable,
51-     "kv.raft_log.non_blocking_synchronization.enabled",

closedts/setting.go
21:     settings.TenantWritable,
22-     "kv.closed_timestamp.target_duration",
--
30:     settings.TenantWritable,
31-     "kv.closed_timestamp.side_transport_interval",
--
43:     settings.TenantWritable,
44-     "kv.closed_timestamp.lead_for_global_reads_override",

queue.go
56:     settings.TenantWritable,
57-     "kv.queue.process.guaranteed_time_budget",

store.go
135:    settings.TenantWritable,
136-    "kv.bulk_io_write.max_rate",
--
143:    settings.TenantWritable,
144-    "kv.bulk_io_write.concurrent_addsstable_requests",
--
156:    settings.TenantWritable,
157-    "kv.bulk_io_write.concurrent_addsstable_as_writes_requests",
--
165:    settings.TenantWritable,
166-    "kv.rangefeed.concurrent_catchup_iterators",
--
175:    settings.TenantWritable,
176-    "kv.store.system_config_update.queue_add_rate",
--
186:    settings.TenantWritable,
187-    "kv.store.system_config_update.queue_add_burst",
--
196:            settings.TenantWritable,
197-            leaseTransferWaitSettingName,
--
224:    settings.TenantWritable,
225-    "kv.bulk_io_write.concurrent_export_requests",

replica_rangefeed.go
46:     settings.TenantWritable,
47-     "kv.rangefeed.enabled",
--
55:     settings.TenantWritable,
56-     "kv.rangefeed.closed_timestamp_refresh_interval",

replica_send.go
41:     settings.TenantWritable,
42-     "kv.concurrency.optimistic_eval_limited_scans.enabled",
```

</details>

Jira issue: CRDB-25213

	// Minimum time interval between system config updates which will lead to
	// enqueuing replicas.
	var queueAdditionOnSystemConfigUpdateRate = settings.RegisterFloatSetting(
	settings.TenantWritable,

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

kvserver: update distribution cluster settings to be system only #98347

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

	var FailedReservationsTimeout = settings.RegisterDurationSetting(
	settings.TenantWritable,
	"server.failed_reservation_timeout",

	var TimeAfterStoreSuspect = settings.RegisterDurationSetting(
	settings.TenantWritable,

	var TimeUntilStoreDead = func() *settings.DurationSetting {
	s := settings.RegisterDurationSetting(
	settings.TenantWritable,

	var AddSSTableRequestSizeFactor = settings.RegisterIntSetting(
	settings.TenantWritable,
	"kv.replica_stats.addsst_request_size_factor",

	// SplitByLoadMergeDelay wraps "kv.range_split.by_load_merge_delay".
	var SplitByLoadMergeDelay = settings.RegisterDurationSetting(
	settings.TenantWritable,
	"kv.range_split.by_load_merge_delay",

	var SplitByLoadEnabled = settings.RegisterBoolSetting(
	settings.TenantWritable,
	"kv.range_split.by_load_enabled",

	var SplitByLoadQPSThreshold = settings.RegisterIntSetting(
	settings.TenantWritable,
	"kv.range_split.load_qps_threshold",

	var SplitByLoadCPUThreshold = settings.RegisterDurationSetting(
	settings.TenantWritable,
	"kv.range_split.load_cpu_threshold",

	var queueGuaranteedProcessingTimeBudget = settings.RegisterDurationSetting(
	settings.TenantWritable,

	var queueAdditionOnSystemConfigUpdateBurst = settings.RegisterIntSetting(
	settings.TenantWritable,
	"kv.store.system_config_update.queue_add_burst",

	var leaseTransferWait = func() *settings.DurationSetting {
	s := settings.RegisterDurationSetting(
	settings.TenantWritable,

kvserver: update distribution cluster settings to be system only #98347

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions