Skip to content

Commit e429795

Browse files
mashhurskaisecheng
mashhurs
and
kaisecheng
authored
Save name came through ENV vars to let Logstash decide using value from either keystore or ENV. (#16026)
* Save name came through ENV vars to let Logstash decide using either keystore or ENV value. * Apply suggestions from code review to simplify array declaration. Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --------- Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com>
1 parent e8597cb commit e429795

1 file changed

Lines changed: 113 additions & 124 deletions

File tree

‎docker/data/logstash/env2yaml/env2yaml.go‎

Lines changed: 113 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,116 @@ import (
2121
"log"
2222
"os"
2323
"strings"
24+
"fmt"
2425
)
2526

26-
// If the given string can be parsed as YAML, then do so and return the
27-
// resulting entity. Otherwise, return the string unmodified.
28-
func FromYamlIfPossible(str string) interface{} {
29-
var entity interface{}
30-
err := yaml.Unmarshal([]byte(str), &entity)
31-
if err == nil {
32-
return entity
33-
} else {
34-
return str
35-
}
27+
var validSettings = []string{
28+
"api.enabled",
29+
"api.http.host",
30+
"api.http.port",
31+
"api.environment",
32+
"node.name",
33+
"path.data",
34+
"pipeline.id",
35+
"pipeline.workers",
36+
"pipeline.output.workers",
37+
"pipeline.batch.size",
38+
"pipeline.batch.delay",
39+
"pipeline.unsafe_shutdown",
40+
"pipeline.ecs_compatibility",
41+
"pipeline.ordered",
42+
"pipeline.plugin_classloaders",
43+
"pipeline.separate_logs",
44+
"path.config",
45+
"config.string",
46+
"config.test_and_exit",
47+
"config.reload.automatic",
48+
"config.reload.interval",
49+
"config.debug",
50+
"config.support_escapes",
51+
"config.field_reference.escape_style",
52+
"event_api.tags.illegal",
53+
"queue.type",
54+
"path.queue",
55+
"queue.page_capacity",
56+
"queue.max_events",
57+
"queue.max_bytes",
58+
"queue.checkpoint.acks",
59+
"queue.checkpoint.writes",
60+
"queue.checkpoint.interval",
61+
"queue.drain",
62+
"dead_letter_queue.enable",
63+
"dead_letter_queue.max_bytes",
64+
"dead_letter_queue.flush_interval",
65+
"dead_letter_queue.storage_policy",
66+
"dead_letter_queue.retain.age",
67+
"path.dead_letter_queue",
68+
"http.enabled", // DEPRECATED: prefer `api.enabled`
69+
"http.environment", // DEPRECATED: prefer `api.environment`
70+
"http.host", // DEPRECATED: prefer `api.http.host`
71+
"http.port", // DEPRECATED: prefer `api.http.port`
72+
"log.level",
73+
"log.format",
74+
"modules",
75+
"metric.collect",
76+
"path.logs",
77+
"path.plugins",
78+
"api.auth.type",
79+
"api.auth.basic.username",
80+
"api.auth.basic.password",
81+
"api.auth.basic.password_policy.mode",
82+
"api.auth.basic.password_policy.length.minimum",
83+
"api.auth.basic.password_policy.include.upper",
84+
"api.auth.basic.password_policy.include.lower",
85+
"api.auth.basic.password_policy.include.digit",
86+
"api.auth.basic.password_policy.include.symbol",
87+
"allow_superuser",
88+
"monitoring.cluster_uuid",
89+
"xpack.monitoring.enabled",
90+
"xpack.monitoring.collection.interval",
91+
"xpack.monitoring.elasticsearch.hosts",
92+
"xpack.monitoring.elasticsearch.username",
93+
"xpack.monitoring.elasticsearch.password",
94+
"xpack.monitoring.elasticsearch.proxy",
95+
"xpack.monitoring.elasticsearch.api_key",
96+
"xpack.monitoring.elasticsearch.cloud_auth",
97+
"xpack.monitoring.elasticsearch.cloud_id",
98+
"xpack.monitoring.elasticsearch.sniffing",
99+
"xpack.monitoring.elasticsearch.ssl.certificate_authority",
100+
"xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint",
101+
"xpack.monitoring.elasticsearch.ssl.verification_mode",
102+
"xpack.monitoring.elasticsearch.ssl.truststore.path",
103+
"xpack.monitoring.elasticsearch.ssl.truststore.password",
104+
"xpack.monitoring.elasticsearch.ssl.keystore.path",
105+
"xpack.monitoring.elasticsearch.ssl.keystore.password",
106+
"xpack.monitoring.elasticsearch.ssl.certificate",
107+
"xpack.monitoring.elasticsearch.ssl.key",
108+
"xpack.monitoring.elasticsearch.ssl.cipher_suites",
109+
"xpack.management.enabled",
110+
"xpack.management.logstash.poll_interval",
111+
"xpack.management.pipeline.id",
112+
"xpack.management.elasticsearch.hosts",
113+
"xpack.management.elasticsearch.username",
114+
"xpack.management.elasticsearch.password",
115+
"xpack.management.elasticsearch.proxy",
116+
"xpack.management.elasticsearch.api_key",
117+
"xpack.management.elasticsearch.cloud_auth",
118+
"xpack.management.elasticsearch.cloud_id",
119+
"xpack.management.elasticsearch.sniffing",
120+
"xpack.management.elasticsearch.ssl.certificate_authority",
121+
"xpack.management.elasticsearch.ssl.ca_trusted_fingerprint",
122+
"xpack.management.elasticsearch.ssl.verification_mode",
123+
"xpack.management.elasticsearch.ssl.truststore.path",
124+
"xpack.management.elasticsearch.ssl.truststore.password",
125+
"xpack.management.elasticsearch.ssl.keystore.path",
126+
"xpack.management.elasticsearch.ssl.keystore.password",
127+
"xpack.management.elasticsearch.ssl.certificate",
128+
"xpack.management.elasticsearch.ssl.key",
129+
"xpack.management.elasticsearch.ssl.cipher_suites",
130+
"xpack.geoip.download.endpoint",
131+
"xpack.geoip.downloader.enabled",
132+
"cloud.id",
133+
"cloud.auth",
36134
}
37135

38136
// Given a setting name, return a downcased version with delimiters removed.
@@ -46,118 +144,9 @@ func squashSetting(setting string) string {
46144
// Given a setting name like "pipeline.workers" or "PIPELINE_UNSAFE_SHUTDOWN",
47145
// return the canonical setting name. eg. 'pipeline.unsafe_shutdown'
48146
func normalizeSetting(setting string) (string, error) {
49-
valid_settings := []string{
50-
"api.enabled",
51-
"api.http.host",
52-
"api.http.port",
53-
"api.environment",
54-
"node.name",
55-
"path.data",
56-
"pipeline.id",
57-
"pipeline.workers",
58-
"pipeline.output.workers",
59-
"pipeline.batch.size",
60-
"pipeline.batch.delay",
61-
"pipeline.unsafe_shutdown",
62-
"pipeline.ecs_compatibility",
63-
"pipeline.ordered",
64-
"pipeline.plugin_classloaders",
65-
"pipeline.separate_logs",
66-
"path.config",
67-
"config.string",
68-
"config.test_and_exit",
69-
"config.reload.automatic",
70-
"config.reload.interval",
71-
"config.debug",
72-
"config.support_escapes",
73-
"config.field_reference.escape_style",
74-
"event_api.tags.illegal",
75-
"queue.type",
76-
"path.queue",
77-
"queue.page_capacity",
78-
"queue.max_events",
79-
"queue.max_bytes",
80-
"queue.checkpoint.acks",
81-
"queue.checkpoint.writes",
82-
"queue.checkpoint.interval",
83-
"queue.drain",
84-
"dead_letter_queue.enable",
85-
"dead_letter_queue.max_bytes",
86-
"dead_letter_queue.flush_interval",
87-
"dead_letter_queue.storage_policy",
88-
"dead_letter_queue.retain.age",
89-
"path.dead_letter_queue",
90-
"http.enabled", // DEPRECATED: prefer `api.enabled`
91-
"http.environment", // DEPRECATED: prefer `api.environment`
92-
"http.host", // DEPRECATED: prefer `api.http.host`
93-
"http.port", // DEPRECATED: prefer `api.http.port`
94-
"log.level",
95-
"log.format",
96-
"modules",
97-
"metric.collect",
98-
"path.logs",
99-
"path.plugins",
100-
"api.auth.type",
101-
"api.auth.basic.username",
102-
"api.auth.basic.password",
103-
"api.auth.basic.password_policy.mode",
104-
"api.auth.basic.password_policy.length.minimum",
105-
"api.auth.basic.password_policy.include.upper",
106-
"api.auth.basic.password_policy.include.lower",
107-
"api.auth.basic.password_policy.include.digit",
108-
"api.auth.basic.password_policy.include.symbol",
109-
"allow_superuser",
110-
"monitoring.cluster_uuid",
111-
"xpack.monitoring.enabled",
112-
"xpack.monitoring.collection.interval",
113-
"xpack.monitoring.elasticsearch.hosts",
114-
"xpack.monitoring.elasticsearch.username",
115-
"xpack.monitoring.elasticsearch.password",
116-
"xpack.monitoring.elasticsearch.proxy",
117-
"xpack.monitoring.elasticsearch.api_key",
118-
"xpack.monitoring.elasticsearch.cloud_auth",
119-
"xpack.monitoring.elasticsearch.cloud_id",
120-
"xpack.monitoring.elasticsearch.sniffing",
121-
"xpack.monitoring.elasticsearch.ssl.certificate_authority",
122-
"xpack.monitoring.elasticsearch.ssl.ca_trusted_fingerprint",
123-
"xpack.monitoring.elasticsearch.ssl.verification_mode",
124-
"xpack.monitoring.elasticsearch.ssl.truststore.path",
125-
"xpack.monitoring.elasticsearch.ssl.truststore.password",
126-
"xpack.monitoring.elasticsearch.ssl.keystore.path",
127-
"xpack.monitoring.elasticsearch.ssl.keystore.password",
128-
"xpack.monitoring.elasticsearch.ssl.certificate",
129-
"xpack.monitoring.elasticsearch.ssl.key",
130-
"xpack.monitoring.elasticsearch.ssl.cipher_suites",
131-
"xpack.management.enabled",
132-
"xpack.management.logstash.poll_interval",
133-
"xpack.management.pipeline.id",
134-
"xpack.management.elasticsearch.hosts",
135-
"xpack.management.elasticsearch.username",
136-
"xpack.management.elasticsearch.password",
137-
"xpack.management.elasticsearch.proxy",
138-
"xpack.management.elasticsearch.api_key",
139-
"xpack.management.elasticsearch.cloud_auth",
140-
"xpack.management.elasticsearch.cloud_id",
141-
"xpack.management.elasticsearch.sniffing",
142-
"xpack.management.elasticsearch.ssl.certificate_authority",
143-
"xpack.management.elasticsearch.ssl.ca_trusted_fingerprint",
144-
"xpack.management.elasticsearch.ssl.verification_mode",
145-
"xpack.management.elasticsearch.ssl.truststore.path",
146-
"xpack.management.elasticsearch.ssl.truststore.password",
147-
"xpack.management.elasticsearch.ssl.keystore.path",
148-
"xpack.management.elasticsearch.ssl.keystore.password",
149-
"xpack.management.elasticsearch.ssl.certificate",
150-
"xpack.management.elasticsearch.ssl.key",
151-
"xpack.management.elasticsearch.ssl.cipher_suites",
152-
"xpack.geoip.download.endpoint",
153-
"xpack.geoip.downloader.enabled",
154-
"cloud.id",
155-
"cloud.auth",
156-
}
157-
158-
for _, valid_setting := range valid_settings {
159-
if squashSetting(setting) == squashSetting(valid_setting) {
160-
return valid_setting, nil
147+
for _, validSetting := range validSettings {
148+
if squashSetting(setting) == squashSetting(validSetting) {
149+
return validSetting, nil
161150
}
162151
}
163152
return "", errors.New("Invalid setting: " + setting)
@@ -186,12 +175,12 @@ func main() {
186175
for _, line := range os.Environ() {
187176
kv := strings.SplitN(line, "=", 2)
188177
key := kv[0]
189-
value := kv[1]
190178
setting, err := normalizeSetting(key)
191179
if err == nil {
192180
foundNewSettings = true
193181
log.Printf("Setting '%s' from environment.", setting)
194-
settings[setting] = FromYamlIfPossible(value)
182+
// we need to keep ${KEY} in the logstash.yml to let Logstash decide using ${KEY}'s value from either keystore or environment
183+
settings[setting] = fmt.Sprintf("${%s}", key)
195184
}
196185
}
197186

0 commit comments

Comments
 (0)