Skip to content

Commit 3d387bd

Browse files
disqdisq
authored
feat!: Remove drift (#887)
Co-authored-by: Kemal Hadimli <disq@users.noreply.github.com>
1 parent bda1476 commit 3d387bd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+497
-5056
lines changed

‎.github/workflows/sanity.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,4 +66,4 @@ jobs:
6666
run: go build -v .
6767

6868
- name: Sanity
69-
run: ./scripts/test-sanity.sh
69+
run: ./scripts/test-sanity.sh

‎cmd/drift.go‎

Lines changed: 0 additions & 63 deletions
This file was deleted.

‎cmd/fetch.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,8 @@ var fetchCmd = &cobra.Command{
2222
Example: ` # Fetch configured providers to PostgreSQL as configured in config.hcl
2323
cloudquery fetch`,
2424
RunE: func(cmd *cobra.Command, args []string) error {
25-
cfgPath := viper.GetString("configPath")
2625
cfgMutator := filterConfigProviders(args)
27-
c, err := console.CreateClient(cmd.Context(), cfgPath, false, cfgMutator, instanceId)
26+
c, err := console.CreateClient(cmd.Context(), getConfigFile(), false, cfgMutator, instanceId)
2827
if err != nil {
2928
return err
3029
}

‎cmd/init.go‎

Lines changed: 138 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@ import (
44
"bytes"
55
"context"
66
"fmt"
7-
"os"
7+
"strings"
88

9+
"github.com/cloudquery/cloudquery/internal/file"
910
"github.com/cloudquery/cloudquery/pkg/config"
1011
"github.com/cloudquery/cloudquery/pkg/core"
11-
"github.com/cloudquery/cloudquery/pkg/module"
12-
"github.com/cloudquery/cloudquery/pkg/module/drift"
1312
"github.com/cloudquery/cloudquery/pkg/plugin/registry"
1413
"github.com/cloudquery/cloudquery/pkg/ui"
1514
"github.com/cloudquery/cloudquery/pkg/ui/console"
15+
"github.com/cloudquery/cq-provider-sdk/cqproto"
1616
"github.com/cloudquery/cq-provider-sdk/provider/diag"
1717
"github.com/google/uuid"
1818
"github.com/hashicorp/hcl/v2/gohcl"
@@ -21,6 +21,7 @@ import (
2121
"github.com/spf13/afero"
2222
"github.com/spf13/cobra"
2323
"github.com/spf13/viper"
24+
"gopkg.in/yaml.v3"
2425
)
2526

2627
const initHelpMsg = "Generate initial config.hcl for fetch command"
@@ -46,14 +47,13 @@ var (
4647
func Initialize(ctx context.Context, providers []string) error {
4748
fs := afero.NewOsFs()
4849

49-
configPath := viper.GetString("configPath")
50+
configPath := getConfigFile() // by definition, this will get us an existing file if possible
5051

5152
if info, _ := fs.Stat(configPath); info != nil {
5253
ui.ColorizedOutput(ui.ColorError, "Error: Config file %s already exists\n", configPath)
5354
return diag.FromError(fmt.Errorf("config file %q already exists", configPath), diag.USER)
5455
}
55-
f := hclwrite.NewEmptyFile()
56-
rootBody := f.Body()
56+
5757
requiredProviders := make([]*config.RequiredProvider, len(providers))
5858
for i, p := range providers {
5959
organization, providerName, provVersion, err := registry.ParseProviderNameWithVersion(p)
@@ -71,18 +71,112 @@ func Initialize(ctx context.Context, providers []string) error {
7171
requiredProviders[i] = &rp
7272
providers[i] = providerName // overwrite "provider@version" with just "provider"
7373
}
74-
// TODO: build this manually with block and add comments as well
75-
cqBlock := gohcl.EncodeAsBlock(&config.CloudQuery{
76-
Providers: requiredProviders,
77-
Connection: &config.Connection{
78-
Username: "postgres",
79-
Password: "pass",
80-
Host: "localhost",
81-
Port: 5432,
82-
Database: "postgres",
83-
SSLMode: "disable",
74+
75+
mainConfig := config.Config{
76+
CloudQuery: config.CloudQuery{
77+
Providers: requiredProviders,
78+
Connection: &config.Connection{
79+
Username: "postgres",
80+
Password: "pass",
81+
Host: "localhost",
82+
Port: 5432,
83+
Database: "postgres",
84+
SSLMode: "disable",
85+
},
86+
},
87+
}
88+
if diags := config.ValidateCQBlock(&mainConfig.CloudQuery); diags.HasErrors() {
89+
return diags
90+
}
91+
92+
cCfg := mainConfig
93+
cCfg.CloudQuery.Connection.DSN = "" // Don't connect
94+
c, err := console.CreateClientFromConfig(ctx, &cCfg, uuid.Nil)
95+
if err != nil {
96+
return err
97+
}
98+
defer c.Close()
99+
if err := c.DownloadProviders(ctx); err != nil {
100+
return err
101+
}
102+
103+
var b []byte
104+
if config.IsNameYAML(configPath) {
105+
b, err = generateYAMLConfig(ctx, c, providers, mainConfig)
106+
} else {
107+
b, err = generateHCLConfig(ctx, c, providers, mainConfig)
108+
}
109+
if err != nil {
110+
return err
111+
}
112+
_ = afero.WriteFile(fs, configPath, b, 0644)
113+
ui.ColorizedOutput(ui.ColorSuccess, "configuration generated successfully to %s\n", configPath)
114+
return nil
115+
}
116+
117+
func generateYAMLConfig(ctx context.Context, c *console.Client, providers []string, mainConfig config.Config) ([]byte, error) {
118+
cqConfig := struct {
119+
CloudQuery config.CloudQuery `yaml:"cloudquery" json:"cloudquery"`
120+
}{
121+
CloudQuery: mainConfig.CloudQuery,
122+
}
123+
b, err := yaml.Marshal(cqConfig)
124+
if err != nil {
125+
return nil, diag.WrapError(err)
126+
}
127+
128+
var cqConfigRaw = struct {
129+
CQ yaml.Node `yaml:"cloudquery"`
130+
}{}
131+
if err := yaml.Unmarshal(b, &cqConfigRaw); err != nil {
132+
return nil, diag.WrapError(err)
133+
}
134+
135+
provNode := &yaml.Node{
136+
Kind: yaml.MappingNode,
137+
HeadComment: "provider configurations",
138+
}
139+
140+
for _, p := range providers {
141+
pCfg, diags := core.GetProviderConfiguration(ctx, c.PluginManager, &core.GetProviderConfigOptions{
142+
Provider: c.ConvertRequiredToRegistry(p),
143+
Format: cqproto.ConfigYAML,
144+
})
145+
if pCfg != nil && pCfg.Format != cqproto.ConfigYAML {
146+
diags = diags.Add(diag.FromError(fmt.Errorf("provider %s doesn't support YAML config. Fallback to HCL or upgrade provider", p), diag.USER, diag.WithDetails("Use `cloudquery init <providers> --config config.hcl` to use HCL config format")))
147+
}
148+
if diags.HasErrors() {
149+
return nil, diags
150+
}
151+
152+
var yCfg yaml.Node
153+
if err := yaml.Unmarshal(pCfg.Config, &yCfg); err != nil {
154+
return nil, diag.WrapError(err)
155+
}
156+
157+
provNode.Content = append(provNode.Content, &yaml.Node{
158+
Kind: yaml.ScalarNode,
159+
Value: p,
160+
})
161+
provNode.Content = append(provNode.Content, yCfg.Content...)
162+
}
163+
164+
nd := struct {
165+
Data map[string]*yaml.Node `yaml:",inline"`
166+
}{
167+
Data: map[string]*yaml.Node{
168+
"cloudquery": &cqConfigRaw.CQ,
169+
"providers": provNode,
84170
},
85-
}, "cloudquery")
171+
}
172+
173+
return yaml.Marshal(&nd)
174+
}
175+
176+
func generateHCLConfig(ctx context.Context, c *console.Client, providers []string, mainConfig config.Config) ([]byte, error) {
177+
f := hclwrite.NewEmptyFile()
178+
rootBody := f.Body()
179+
cqBlock := gohcl.EncodeAsBlock(&mainConfig.CloudQuery, "cloudquery")
86180

87181
// Remove deprecated "plugin_directory" and "policy_directory"
88182
cqBlock.Body().RemoveAttribute("plugin_directory")
@@ -97,22 +191,6 @@ func Initialize(ctx context.Context, providers []string) error {
97191
}
98192

99193
rootBody.AppendBlock(cqBlock)
100-
cfg, diags := config.NewParser(
101-
config.WithEnvironmentVariables(config.EnvVarPrefix, os.Environ()),
102-
).LoadConfigFromSource("init.hcl", f.Bytes())
103-
if diags.HasErrors() {
104-
return diags
105-
}
106-
107-
cfg.CloudQuery.Connection.DSN = "" // Don't connect
108-
c, err := console.CreateClientFromConfig(ctx, cfg, uuid.Nil)
109-
if err != nil {
110-
return err
111-
}
112-
defer c.Close()
113-
if err := c.DownloadProviders(ctx); err != nil {
114-
return err
115-
}
116194
rootBody.AppendNewline()
117195
rootBody.AppendUnstructuredTokens(hclwrite.Tokens{
118196
{
@@ -124,37 +202,47 @@ func Initialize(ctx context.Context, providers []string) error {
124202
var buffer bytes.Buffer
125203
buffer.WriteString("// Configuration AutoGenerated by CloudQuery CLI\n")
126204
if n, err := buffer.Write(f.Bytes()); n != len(f.Bytes()) || err != nil {
127-
return err
205+
return nil, err
128206
}
129207
for _, p := range providers {
130208
pCfg, diags := core.GetProviderConfiguration(ctx, c.PluginManager, &core.GetProviderConfigOptions{
131209
Provider: c.ConvertRequiredToRegistry(p),
210+
Format: cqproto.ConfigHCL,
132211
})
133-
212+
if pCfg != nil && pCfg.Format != cqproto.ConfigHCL {
213+
diags = diags.Add(diag.FromError(fmt.Errorf("provider %s doesn't support HCL config. Please upgrade provider", p), diag.USER))
214+
}
134215
if diags.HasErrors() {
135-
return diags
216+
return nil, diags
136217
}
137218
buffer.Write(pCfg.Config)
138219
buffer.WriteString("\n")
139220
}
140-
mm := module.NewManager(nil, nil)
141-
mm.Register(drift.New())
142-
if mex := mm.ExampleConfigs(providers); len(mex) > 0 {
143-
buffer.WriteString("\n// Module Configurations\nmodules {\n")
144-
for _, c := range mex {
145-
buffer.WriteString(c)
146-
buffer.WriteString("\n")
147-
}
148-
buffer.WriteString("}\n")
149-
}
150221

151-
formattedData := hclwrite.Format(buffer.Bytes())
152-
_ = afero.WriteFile(fs, configPath, formattedData, 0644)
153-
ui.ColorizedOutput(ui.ColorSuccess, "configuration generated successfully to %s\n", configPath)
154-
return nil
222+
return hclwrite.Format(buffer.Bytes()), nil
155223
}
156224

157225
func init() {
158226
initCmd.SetUsageTemplate(usageTemplateWithFlags)
159227
rootCmd.AddCommand(initCmd)
160228
}
229+
230+
// getConfigFile returns the config filename
231+
// if it ends with ".*", .hcl and .yml extensions are tried in order to find the existing file, if available
232+
func getConfigFile() string {
233+
configPath := viper.GetString("configPath")
234+
if !strings.HasSuffix(configPath, ".*") {
235+
return configPath
236+
}
237+
238+
fs := file.NewOsFs()
239+
noSuffix := strings.TrimSuffix(configPath, ".*")
240+
for _, tryExt := range []string{".hcl", ".yml"} {
241+
tryFn := noSuffix + tryExt
242+
if _, err := fs.Stat(tryFn); err == nil {
243+
return tryFn
244+
}
245+
}
246+
247+
return noSuffix + ".hcl"
248+
}

0 commit comments

Comments
 (0)