Skip to content

Commit c225107

Browse files
disqdisq
authored
feat: Add install command to pre-install required plugins (#13941)
Fixes #13937 Requires cloudquery/plugin-pb-go#103 (merged/tagged)
1 parent 4fa9ef3 commit c225107

6 files changed

Lines changed: 239 additions & 0 deletions

File tree

‎cli/cmd/doc_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ var docFiles = []string{
1111
"cloudquery_sync.md",
1212
"cloudquery_migrate.md",
1313
"cloudquery_tables.md",
14+
"cloudquery_install.md",
1415
}
1516

1617
func TestDoc(t *testing.T) {

‎cli/cmd/install.go‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
package cmd
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/cloudquery/cloudquery/cli/internal/specs/v0"
8+
"github.com/cloudquery/plugin-pb-go/managedplugin"
9+
"github.com/rs/zerolog/log"
10+
"github.com/spf13/cobra"
11+
)
12+
13+
const (
14+
installShort = "Install required plugin images from your configuration"
15+
installExample = `# Install required plugins specified in directory
16+
cloudquery install ./directory
17+
# Install required plugins specified in directory and config files
18+
cloudquery install ./directory ./aws.yml ./pg.yml
19+
`
20+
)
21+
22+
func newCmdInstall() *cobra.Command {
23+
cmd := &cobra.Command{
24+
Use: "install [files or directories]",
25+
Short: installShort,
26+
Long: installShort,
27+
Example: installExample,
28+
Args: cobra.MinimumNArgs(1),
29+
RunE: install,
30+
}
31+
return cmd
32+
}
33+
34+
func install(cmd *cobra.Command, args []string) error {
35+
cqDir, err := cmd.Flags().GetString("cq-dir")
36+
if err != nil {
37+
return err
38+
}
39+
40+
ctx := cmd.Context()
41+
log.Info().Strs("args", args).Msg("Loading spec(s)")
42+
fmt.Printf("Loading spec(s) from %s\n", strings.Join(args, ", "))
43+
specReader, err := specs.NewSpecReader(args)
44+
if err != nil {
45+
return fmt.Errorf("failed to load spec(s) from %s. Error: %w", strings.Join(args, ", "), err)
46+
}
47+
sources := specReader.Sources
48+
destinations := specReader.Destinations
49+
opts := []managedplugin.Option{managedplugin.WithNoExec()}
50+
if cqDir != "" {
51+
opts = append(opts, managedplugin.WithDirectory(cqDir))
52+
}
53+
54+
sourcePluginConfigs := make([]managedplugin.Config, 0, len(sources))
55+
for _, source := range sources {
56+
sourcePluginConfigs = append(sourcePluginConfigs, managedplugin.Config{
57+
Name: source.Name,
58+
Version: source.Version,
59+
Path: source.Path,
60+
Registry: SpecRegistryToPlugin(source.Registry),
61+
})
62+
}
63+
destinationPluginConfigs := make([]managedplugin.Config, 0, len(destinations))
64+
for _, destination := range destinations {
65+
destinationPluginConfigs = append(destinationPluginConfigs, managedplugin.Config{
66+
Name: destination.Name,
67+
Version: destination.Version,
68+
Path: destination.Path,
69+
Registry: SpecRegistryToPlugin(destination.Registry),
70+
})
71+
}
72+
73+
if _, err := managedplugin.NewClients(ctx, managedplugin.PluginSource, sourcePluginConfigs, opts...); err != nil {
74+
return err
75+
}
76+
if _, err := managedplugin.NewClients(ctx, managedplugin.PluginDestination, destinationPluginConfigs, opts...); err != nil {
77+
return err
78+
}
79+
80+
return nil
81+
}

‎cli/cmd/install_test.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package cmd
2+
3+
import (
4+
"os"
5+
"path"
6+
"runtime"
7+
"sort"
8+
"strings"
9+
"testing"
10+
11+
"github.com/stretchr/testify/assert"
12+
"github.com/stretchr/testify/require"
13+
)
14+
15+
func TestInstall(t *testing.T) {
16+
configs := []struct {
17+
name string
18+
config string
19+
wantSourceFiles int
20+
wantDestFiles int
21+
}{
22+
{
23+
name: "sync_success_sourcev1_destv0",
24+
config: "sync-success-sourcev1-destv0.yml",
25+
wantSourceFiles: 2,
26+
wantDestFiles: 2,
27+
},
28+
{
29+
name: "multiple_sources",
30+
config: "multiple-sources.yml",
31+
wantSourceFiles: 2,
32+
wantDestFiles: 2,
33+
},
34+
{
35+
name: "multiple_destinations",
36+
config: "multiple-destinations.yml",
37+
wantSourceFiles: 2,
38+
wantDestFiles: 4,
39+
},
40+
{
41+
name: "multiple_sources_destinations",
42+
config: "multiple-sources-destinations.yml",
43+
wantSourceFiles: 2,
44+
wantDestFiles: 2,
45+
},
46+
}
47+
_, filename, _, _ := runtime.Caller(0)
48+
currentDir := path.Dir(filename)
49+
50+
for _, tc := range configs {
51+
tc := tc
52+
t.Run(tc.name, func(t *testing.T) {
53+
cqDir := t.TempDir()
54+
t.Cleanup(func() {
55+
CloseLogFile()
56+
os.RemoveAll(cqDir)
57+
})
58+
testConfig := path.Join(currentDir, "testdata", tc.config)
59+
logFileName := path.Join(cqDir, "cloudquery.log")
60+
cmd := NewCmdRoot()
61+
cmd.SetArgs([]string{"install", testConfig, "--cq-dir", cqDir, "--log-file-name", logFileName})
62+
err := cmd.Execute()
63+
assert.NoError(t, err)
64+
65+
// check if all files were created
66+
justFiles := readFiles(t, cqDir, "")
67+
68+
sourceFiles, destFiles := 0, 0
69+
for _, file := range justFiles {
70+
if strings.HasPrefix(file, "plugins/source") {
71+
sourceFiles++
72+
} else if strings.HasPrefix(file, "plugins/destination") {
73+
destFiles++
74+
}
75+
}
76+
assert.Equalf(t, tc.wantSourceFiles, sourceFiles, "expected %d source files, got %d", tc.wantSourceFiles, sourceFiles)
77+
assert.Equalf(t, tc.wantDestFiles, destFiles, "expected %d destination files, got %d", tc.wantDestFiles, destFiles)
78+
if t.Failed() {
79+
t.Logf("files found: %v", justFiles)
80+
t.FailNow()
81+
}
82+
83+
// check that log was written and contains some lines
84+
b, logFileError := os.ReadFile(path.Join(cqDir, "cloudquery.log"))
85+
logContent := string(b)
86+
require.NoError(t, logFileError, "failed to read cloudquery.log")
87+
require.NotEmpty(t, logContent, "cloudquery.log empty; expected some logs")
88+
})
89+
}
90+
}
91+
92+
func readFiles(t *testing.T, basedir, prefix string) []string {
93+
files, err := os.ReadDir(basedir)
94+
assert.NoError(t, err)
95+
var justFiles []string
96+
for i := range files {
97+
name := files[i].Name()
98+
99+
if !files[i].IsDir() {
100+
justFiles = append(justFiles, path.Join(prefix, name))
101+
continue
102+
}
103+
104+
justFiles = append(justFiles, readFiles(t, path.Join(basedir, files[i].Name()), path.Join(prefix, name))...)
105+
}
106+
sort.Strings(justFiles)
107+
return justFiles
108+
}

‎cli/cmd/root.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ func NewCmdRoot() *cobra.Command {
127127
NewCmdSync(),
128128
NewCmdMigrate(),
129129
newCmdDoc(),
130+
newCmdInstall(),
130131
NewCmdTables(),
131132
newCmdLogin(),
132133
newCmdPublish(),

‎website/pages/docs/reference/cli/cloudquery.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ Find more information at:
2929

3030
### SEE ALSO
3131

32+
* [cloudquery install](/docs/reference/cli/cloudquery_install) - Install required plugin images from your configuration
3233
* [cloudquery migrate](/docs/reference/cli/cloudquery_migrate) - Update schema of your destinations based on the latest changes in sources from your configuration
3334
* [cloudquery sync](/docs/reference/cli/cloudquery_sync) - Sync resources from configured source plugins to destinations
3435
* [cloudquery tables](/docs/reference/cli/cloudquery_tables) - Generate documentation for all supported tables of source plugins specified in the spec(s)

‎website/pages/docs/reference/cli/cloudquery_install.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
---
2+
title: "install"
3+
---
4+
## cloudquery install
5+
6+
Install required plugin images from your configuration
7+
8+
### Synopsis
9+
10+
Install required plugin images from your configuration
11+
12+
```
13+
cloudquery install [files or directories] [flags]
14+
```
15+
16+
### Examples
17+
18+
```
19+
# Install required plugins specified in directory
20+
cloudquery install ./directory
21+
# Install required plugins specified in directory and config files
22+
cloudquery install ./directory ./aws.yml ./pg.yml
23+
24+
```
25+
26+
### Options
27+
28+
```
29+
-h, --help help for install
30+
```
31+
32+
### Options inherited from parent commands
33+
34+
```
35+
--cq-dir string directory to store cloudquery files, such as downloaded plugins (default ".cq")
36+
--log-console enable console logging
37+
--log-file-name string Log filename (default "cloudquery.log")
38+
--log-format string Logging format (json, text) (default "text")
39+
--log-level string Logging level (default "info")
40+
--no-log-file Disable logging to file
41+
--telemetry-level string Telemetry level (none, errors, stats, all) (default "all")
42+
```
43+
44+
### SEE ALSO
45+
46+
* [cloudquery](/docs/reference/cli/cloudquery) - CloudQuery CLI
47+

0 commit comments

Comments
 (0)