Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
125 commits
Select commit Hold shift + click to select a range
e5b34d5
add new policy Filter
bbernays Jan 19, 2022
3d592d7
Add tests for p.Filter
bbernays Jan 19, 2022
85cb429
Support Filtering on describe
bbernays Jan 19, 2022
5f796f2
Update client.go
bbernays Jan 19, 2022
fb55171
Update client.go
bbernays Jan 19, 2022
a668055
Add Support for Check level filter
bbernays Jan 20, 2022
33cdb91
Update execute.go
bbernays Jan 20, 2022
611dbc8
Update manager.go
bbernays Jan 20, 2022
7664fa2
Update execute_test.go
bbernays Jan 20, 2022
048514b
Update policy.go
bbernays Jan 20, 2022
0d81b2f
Update execute.go
bbernays Jan 20, 2022
04702a4
Merge branch 'main' into Remove-filtering-from-executor
bbernays Jan 20, 2022
1a8cacd
Update execute_test.go
bbernays Jan 20, 2022
4dfa7f1
Merge branch 'Remove-filtering-from-executor' of https://github.com/b…
bbernays Jan 20, 2022
d4159cc
Update execute.go
bbernays Jan 20, 2022
2785416
Update manager.go
bbernays Jan 20, 2022
d782e63
Tests pass locally
bbernays Jan 20, 2022
3d215b1
Loca
bbernays Jan 20, 2022
eac026b
Update execute_test.go
bbernays Jan 20, 2022
4a84f41
Update execute.go
bbernays Jan 20, 2022
1475447
Update execute.go
bbernays Jan 20, 2022
3cb27af
Update manager.go
bbernays Jan 20, 2022
f280c6b
Update execute_test.go
bbernays Jan 20, 2022
3e77b68
Create parse.go
bbernays Jan 18, 2022
48a0673
Update go.mod
bbernays Jan 18, 2022
17d0cf6
Update go.sum
bbernays Jan 18, 2022
39e063d
Create database
bbernays Jan 18, 2022
983a3d2
Create manager.go
bbernays Jan 18, 2022
1e56d1e
change package
bbernays Jan 18, 2022
0f37ba0
Update go.mod
bbernays Jan 19, 2022
0b2c2b6
Update go.sum
bbernays Jan 19, 2022
6b59dd4
Create policy_snapshot.go
bbernays Jan 19, 2022
06512e1
Update client.go
bbernays Jan 19, 2022
bcb6636
Create db.go
bbernays Jan 19, 2022
833e10e
Update policy_snapshot.go
bbernays Jan 20, 2022
72deb3d
Update policy.go
bbernays Jan 20, 2022
c6032ca
Update client.go
bbernays Jan 20, 2022
e391aa5
Update policy.go
bbernays Jan 20, 2022
eb5e32b
checkpoint
bbernays Jan 21, 2022
c99612a
discard old format
bbernays Jan 24, 2022
5b89698
Update execute.go
bbernays Jan 24, 2022
73a4cc2
Update snapshots.go
bbernays Jan 24, 2022
29e0d7a
Update snapshots.go
bbernays Jan 24, 2022
54143ce
Update manager.go
bbernays Jan 24, 2022
780f7fb
Update client.go
bbernays Jan 24, 2022
1903671
Update manager.go
bbernays Jan 24, 2022
2c284d9
Update client.go
bbernays Jan 24, 2022
53ad65f
parse dsn
bbernays Jan 24, 2022
8097869
Update snapshots.go
bbernays Jan 24, 2022
8192b8e
Update snapshots.go
bbernays Jan 24, 2022
ddf1616
Dump Views
bbernays Jan 24, 2022
2af12de
Working
bbernays Jan 25, 2022
58f3e02
Update snapshots.go
bbernays Jan 25, 2022
b1cc71f
Update client.go
bbernays Jan 25, 2022
b50d065
Merge branch 'main' into Policy-Testing
bbernays Jan 26, 2022
a8b99a7
Update snapshots.go
bbernays Jan 26, 2022
0bf6afc
Merge branch 'main' into Policy-Testing
bbernays Jan 27, 2022
4b91e32
Update manager.go
bbernays Jan 27, 2022
1b096ed
runner
bbernays Jan 26, 2022
cb47da6
Update helper.go
bbernays Jan 27, 2022
54cdd09
Update helper.go
bbernays Jan 27, 2022
0ccd6ea
Update helper_test.go
bbernays Jan 27, 2022
17429de
Update helper.go
bbernays Jan 27, 2022
269252a
Update snapshots.go
bbernays Jan 27, 2022
76adc13
snapshot
bbernays Jan 27, 2022
cbfd99d
Update snapshots.go
bbernays Jan 28, 2022
57e1475
Update helper.go
bbernays Jan 28, 2022
677ce28
push
bbernays Jan 28, 2022
4dd5995
Update helper.go
bbernays Jan 28, 2022
b90099e
Update helper_test.go
bbernays Jan 28, 2022
3ccfd26
Update helper_test.go
bbernays Jan 28, 2022
d584073
Update helper.go
bbernays Jan 28, 2022
3981028
Update helper.go
bbernays Jan 28, 2022
596339a
Update client.go
bbernays Jan 28, 2022
e19ff91
Merge branch 'cloudquery:main' into Test-Runner
bbernays Jan 28, 2022
d0a8c82
Update client.go
bbernays Jan 28, 2022
2a56f67
Refactor
bbernays Jan 29, 2022
12535bf
Update client.go
bbernays Jan 31, 2022
a42b3da
Update .gitignore
bbernays Feb 1, 2022
d5b0ce5
Update execute.go
bbernays Feb 1, 2022
bd9dd77
Update manager.go
bbernays Feb 1, 2022
69bacec
Update snapshots.go
bbernays Feb 1, 2022
0e9044a
Update helper.go
bbernays Feb 1, 2022
7a8999d
Update helper_test.go
bbernays Feb 1, 2022
9842897
Merge branch 'main' into Test-Runner
bbernays Feb 1, 2022
59451e7
Use new version of CloudQuery SDK
bbernays Feb 1, 2022
822975c
use new version of cq-provider-sdk
bbernays Feb 2, 2022
0f85f3e
Make compile
bbernays Feb 2, 2022
bce29a1
refactor
bbernays Feb 2, 2022
e4aa99e
update go mod
bbernays Feb 2, 2022
64757a3
Merge branch 'cloudquery:main' into Test-Runner
bbernays Feb 2, 2022
6b337f5
Update helper_test.go
bbernays Feb 2, 2022
9f69d92
cleanup
bbernays Feb 2, 2022
f0a779d
Merge branch 'main' into Test-Runner
bbernays Feb 3, 2022
241c9bd
Merge branch 'cloudquery:main' into Test-Runner
bbernays Feb 3, 2022
2633b4d
Add `cq policy test` command
bbernays Feb 4, 2022
272f77b
Change path to be selector
bbernays Feb 4, 2022
9492e14
replace temp file
bbernays Feb 4, 2022
7f1e7be
move file to policy package
bbernays Feb 4, 2022
8e62eb1
Update client.go
bbernays Feb 4, 2022
4e4fd83
Update manager.go
bbernays Feb 4, 2022
124dfc7
remove extra args
bbernays Feb 4, 2022
0c76799
Update client.go
bbernays Feb 4, 2022
a2aaae7
Update client.go
bbernays Feb 4, 2022
6505bae
rename
bbernays Feb 4, 2022
9c73a88
Create snapshot_testing_test.go
bbernays Feb 4, 2022
910dc48
Update snapshot_testing.go
bbernays Feb 4, 2022
407cdae
Respond to comments
bbernays Feb 4, 2022
f2adfbf
Update manager.go
bbernays Feb 4, 2022
2a81c91
Update snapshots.go
bbernays Feb 4, 2022
25fde86
Update snapshot_testing.go
bbernays Feb 7, 2022
e3ea87d
Update snapshots.go
bbernays Feb 7, 2022
02ad178
Update snapshots.go
bbernays Feb 7, 2022
5d03336
Update snapshot_testing_test.go
bbernays Feb 7, 2022
3b7814c
Update snapshot_testing_test.go
bbernays Feb 17, 2022
ad1791a
Merge branch 'main' into Test-Runner
bbernays Feb 17, 2022
5d1c7c3
Update drift.go
bbernays Feb 17, 2022
9ca9429
Update policy_snapshot.go
bbernays Feb 17, 2022
e31fdea
Update client.go
bbernays Feb 17, 2022
4b0db2d
Update snapshot_testing_test.go
bbernays Feb 17, 2022
a7da214
Update snapshots.go
bbernays Feb 17, 2022
7fad746
respond to comments
bbernays Feb 17, 2022
0dcf17a
Update snapshot_testing.go
bbernays Feb 17, 2022
f74d34c
Refactor
bbernays Feb 20, 2022
f26043f
nits
bbernays Feb 21, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -21,4 +21,5 @@ config.hcl
config_old.hcl
/pkg/policy/tests/output/
/pkg/policy/cq/*
database-data/*
database-data/*
dest/*
27 changes: 27 additions & 0 deletions cmd/policy_snapshot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package cmd

import (
"context"

"github.com/cloudquery/cloudquery/pkg/ui/console"
"github.com/spf13/cobra"
)

const policySnapshotHelpMsg = `Take database snapshot of all tables included in a CloudQuery policy`

var (
snapshotPolicyCmd = &cobra.Command{
Use: "snapshot",
Short: policySnapshotHelpMsg,
Long: policySnapshotHelpMsg,
Args: cobra.ExactArgs(2),
Run: handleCommand(func(ctx context.Context, c *console.Client, cmd *cobra.Command, args []string) error {
return c.SnapshotPolicy(ctx, args[0], args[1])
}),
}
)

func init() {
snapshotPolicyCmd.SetUsageTemplate(usageTemplateWithFlags)
policyCmd.AddCommand(snapshotPolicyCmd)
}
35 changes: 35 additions & 0 deletions cmd/policy_snapshottest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package cmd

import (
"context"

"github.com/cloudquery/cloudquery/pkg/ui/console"

"github.com/spf13/cobra"
)

const policyTestHelpMsg = "Tests policy against a precompiled set of database snapshots"

var (
policyTestCmd = &cobra.Command{
Use: "test",
Short: policyTestHelpMsg,
Long: policyTestHelpMsg,
Example: `
# Download & Run the policies defined in your config
cloudquery policy test path/to/policy.hcl path/to/snapshot/dir selector
`,
Run: handleCommand(func(ctx context.Context, c *console.Client, cmd *cobra.Command, args []string) error {
return c.TestPolicies(ctx, args[0], args[1])
}),
Args: cobra.ExactArgs(2),
}
)

func init() {
flags := policyTestCmd.Flags()
flags.StringVar(&outputDir, "output-dir", "", "Generates a new file for each policy at the given dir with the output")
flags.BoolVar(&noResults, "no-results", false, "Do not show policies results")
policyRunCmd.SetUsageTemplate(usageTemplateWithFlags)
policyCmd.AddCommand(policyTestCmd)
}
13 changes: 8 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/hashicorp/go-plugin v1.4.3
github.com/hashicorp/go-version v1.3.0
github.com/hashicorp/hcl/v2 v2.10.1
github.com/jackc/pgx/v4 v4.13.0
github.com/jackc/pgx/v4 v4.14.1
github.com/mattn/go-isatty v0.0.14
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/rs/zerolog v1.20.0
Expand Down Expand Up @@ -42,9 +42,10 @@ require (
github.com/google/go-cmp v0.5.7
github.com/google/uuid v1.3.0
github.com/hashicorp/go-getter v1.5.10
github.com/jackc/pgconn v1.10.0
github.com/jackc/pgconn v1.10.1
github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451
github.com/jackc/pgtype v1.8.1
github.com/jackc/pgtype v1.9.1
github.com/jeremywohl/flatten v1.0.1
github.com/lib/pq v1.10.3
github.com/olekukonko/tablewriter v0.0.5
github.com/spf13/cast v1.4.1
Expand Down Expand Up @@ -86,9 +87,9 @@ require (
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.1.1 // indirect
github.com/jackc/pgproto3/v2 v2.2.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/puddle v1.1.4 // indirect
github.com/jackc/puddle v1.2.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jstemmer/go-junit-report v0.9.1 // indirect
github.com/klauspost/compress v1.13.6 // indirect
Expand All @@ -111,6 +112,7 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/smartystreets/goconvey v1.7.2 // indirect
github.com/spf13/jwalterweatherman v1.0.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.3.0 // indirect
Expand All @@ -137,6 +139,7 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
gopkg.in/ini.v1 v1.51.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
Expand Down
26 changes: 18 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -703,8 +703,9 @@ github.com/jackc/pgconn v1.7.0/go.mod h1:sF/lPpNEMEOp+IYhyQGdAvrG20gWf6A1tKlr0v7
github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
github.com/jackc/pgconn v1.10.0 h1:4EYhlDVEMsJ30nNj0mmgwIUXoq7e9sMJrVC2ED6QlCU=
github.com/jackc/pgconn v1.10.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8=
github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 h1:WAvSpGf7MsFuzAtK4Vk7R4EVe+liW4x83r4oWu0WHKw=
github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
Expand All @@ -726,8 +727,9 @@ github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX
github.com/jackc/pgproto3/v2 v2.0.5/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgproto3/v2 v2.0.7/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgproto3/v2 v2.1.1 h1:7PQ/4gLoqnl87ZxL7xjO0DR5gYuviDCZxQJsUlFW1eI=
github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns=
github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
Expand All @@ -741,8 +743,9 @@ github.com/jackc/pgtype v1.3.1-0.20200606141011-f6355165a91c/go.mod h1:cvk9Bgu/V
github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
github.com/jackc/pgtype v1.6.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig=
github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
github.com/jackc/pgtype v1.8.1 h1:9k0IXtdJXHJbyAWQgbWr1lU+MEhPXZz6RIXxfR5oxXs=
github.com/jackc/pgtype v1.8.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
github.com/jackc/pgtype v1.9.1 h1:MJc2s0MFS8C3ok1wQTdQxWuXQcB6+HwAm5x1CzW7mf0=
github.com/jackc/pgtype v1.9.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o=
github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I=
github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
Expand All @@ -755,16 +758,20 @@ github.com/jackc/pgx/v4 v4.6.1-0.20200606145419-4e5062306904/go.mod h1:ZDaNWkt9s
github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0=
github.com/jackc/pgx/v4 v4.10.1/go.mod h1:QlrWebbs3kqEZPHCTGyxecvzG6tvIsYu+A5b1raylkA=
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
github.com/jackc/pgx/v4 v4.13.0 h1:JCjhT5vmhMAf/YwBHLvrBn4OGdIQBiFG6ym8Zmdx570=
github.com/jackc/pgx/v4 v4.13.0/go.mod h1:9P4X524sErlaxj0XSGZk7s+LD0eOyu1ZDUrrpznYDF0=
github.com/jackc/pgx/v4 v4.14.1 h1:71oo1KAGI6mXhLiTMn6iDFcp3e7+zon/capWjl2OEFU=
github.com/jackc/pgx/v4 v4.14.1/go.mod h1:RgDuE4Z34o7XE92RpLsvFiOEfrAUT0Xt2KxvX73W06M=
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.2/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.4 h1:5Ey/o5IfV7dYX6Znivq+N9MdK1S18OJI5OJq6EAAADw=
github.com/jackc/puddle v1.1.4/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.2.0 h1:DNDKdn/pDrWvDWyT2FYvpZVE81OAhWrjCv19I9n108Q=
github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jeremywohl/flatten v1.0.1 h1:LrsxmB3hfwJuE+ptGOijix1PIfOoKLJ3Uee/mzbgtrs=
github.com/jeremywohl/flatten v1.0.1/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ=
github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs=
Expand Down Expand Up @@ -1100,11 +1107,13 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=
github.com/snowflakedb/gosnowflake v1.4.3/go.mod h1:1kyg2XEduwti88V11PKRHImhXLK5WpGiayY6lFNYb98=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
Expand Down Expand Up @@ -1797,8 +1806,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
Expand Down
5 changes: 2 additions & 3 deletions pkg/module/drift/drift.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"regexp"
"strings"

"github.com/cloudquery/cloudquery/pkg/module"
"github.com/cloudquery/cloudquery/pkg/module/drift/terraform"
"github.com/cloudquery/cq-provider-sdk/cqproto"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/doug-martin/goqu/v9"
Expand All @@ -18,9 +20,6 @@ import (
"github.com/jackc/pgconn"
"github.com/jackc/pgerrcode"
"github.com/spf13/afero"

"github.com/cloudquery/cloudquery/pkg/module"
"github.com/cloudquery/cloudquery/pkg/module/drift/terraform"
)

type Drift struct {
Expand Down
7 changes: 6 additions & 1 deletion pkg/module/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ import (
"github.com/hashicorp/go-hclog"
)

type LowLevelQueryExecer interface {
execution.Copier
execution.QueryExecer
}

// ManagerImpl is the manager implementation struct.
type ManagerImpl struct {
modules map[string]Module
Expand Down Expand Up @@ -44,7 +49,7 @@ type moduleInfoRequester interface {
}

// NewManager returns a new manager instance.
func NewManager(pool execution.QueryExecer, logger hclog.Logger, r moduleInfoRequester) *ManagerImpl {
func NewManager(pool LowLevelQueryExecer, logger hclog.Logger, r moduleInfoRequester) *ManagerImpl {
return &ManagerImpl{
modules: make(map[string]Module),
pool: pool,
Expand Down
7 changes: 3 additions & 4 deletions pkg/policy/execute.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"strings"
"time"

"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-version"
"github.com/spf13/afero"
Expand Down Expand Up @@ -45,7 +44,7 @@ func (f Update) DoneCount() int {
// Executor implements the execution framework.
type Executor struct {
// Connection to the database
conn execution.QueryExecer
conn LowLevelQueryExecer
log hclog.Logger

PolicyPath []string
Expand Down Expand Up @@ -101,7 +100,7 @@ type ExecuteRequest struct {
}

// NewExecutor creates a new executor.
func NewExecutor(conn execution.QueryExecer, log hclog.Logger, progressUpdate UpdateCallback) *Executor {
func NewExecutor(conn LowLevelQueryExecer, log hclog.Logger, progressUpdate UpdateCallback) *Executor {
return &Executor{
conn: conn,
log: log,
Expand Down Expand Up @@ -231,7 +230,7 @@ func (e *Executor) executeQuery(ctx context.Context, q *Check) (*QueryResult, er
// createViews creates temporary views for given config.Policy, and any views defined by sub-policies
func (e *Executor) createViews(ctx context.Context, policy *Policy) error {
for _, v := range policy.Views {
e.log.Info("creating policy view", "view", v.Name)
e.log.Info("creating policy view", "view", v.Name, "query", v.Query)
if err := e.conn.Exec(ctx, fmt.Sprintf("CREATE OR REPLACE TEMPORARY VIEW %s AS %s", v.Name, v.Query)); err != nil {
return fmt.Errorf("failed to create view %s/%s: %w", policy.Name, v.Name, err)
}
Expand Down
3 changes: 1 addition & 2 deletions pkg/policy/execute_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,11 @@ import (
"testing"

sdkdb "github.com/cloudquery/cq-provider-sdk/database"
"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/go-hclog"
"github.com/stretchr/testify/assert"
)

func setupPolicyDatabase(t *testing.T, tableName string) (execution.QueryExecer, func(t *testing.T)) {
func setupPolicyDatabase(t *testing.T, tableName string) (LowLevelQueryExecer, func(t *testing.T)) {
conn, err := sdkdb.New(context.Background(), hclog.NewNullLogger(), "postgres://postgres:pass@localhost:5432/postgres")
assert.NoError(t, err)

Expand Down
51 changes: 48 additions & 3 deletions pkg/policy/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,14 @@ package policy
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/cloudquery/cq-provider-sdk/provider/execution"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
uuid "github.com/satori/go.uuid"

"github.com/hashicorp/go-hclog"
)
Expand All @@ -15,13 +19,18 @@ const (
CloudQueryOrg = "cloudquery-policies"
)

type LowLevelQueryExecer interface {
execution.Copier
execution.QueryExecer
}

// ManagerImpl is the manager implementation struct.
type ManagerImpl struct {
// policyDirectory points to the local policy directory
policyDirectory string

// Instance of a database connection pool
pool execution.QueryExecer
pool LowLevelQueryExecer

// Logger instance
logger hclog.Logger
Expand All @@ -35,17 +44,54 @@ type Manager interface {

// Load the policy
Load(ctx context.Context, policy *Policy) (*Policy, error)

// Take a Snapshot of a policy
Snapshot(ctx context.Context, policy *Policy, destination, selector string) error
}

// NewManager returns a new manager instance.
func NewManager(policyDir string, pool execution.QueryExecer, logger hclog.Logger) *ManagerImpl {
func NewManager(policyDir string, pool LowLevelQueryExecer, logger hclog.Logger) *ManagerImpl {
return &ManagerImpl{
policyDirectory: policyDir,
pool: pool,
logger: logger,
}
}

//
func createSnapshotPath(directory, queryName string) (string, error) {
path := strings.TrimSuffix(directory, "/")
cleanedPath := filepath.Join(path, queryName, "tests", uuid.NewV4().String())

err := os.MkdirAll(cleanedPath, os.ModePerm)
if err != nil {
return "", err
}
return cleanedPath, nil
}

func (m *ManagerImpl) Snapshot(ctx context.Context, policy *Policy, outputPath, subpath string) error {
e := NewExecutor(m.pool, m.logger, nil)

if err := e.createViews(ctx, policy); err != nil {
return err
}

tableNames, err := e.extractTableNames(ctx, policy.Checks[0].Query)
if err != nil {
return err
}
snapShotPath, err := createSnapshotPath(outputPath, subpath)
if err != nil {
return err
}
err = StoreSnapshot(ctx, e, snapShotPath, tableNames)
if err != nil {
return err
}

return StoreOutput(ctx, e, policy, snapShotPath)
}
func (m *ManagerImpl) Load(ctx context.Context, policy *Policy) (*Policy, error) {
var err error
// if policy is configured with source we load it first
Expand Down Expand Up @@ -106,7 +152,6 @@ func (m *ManagerImpl) Run(ctx context.Context, request *ExecuteRequest) (*Execut
})
}
}

// execute the queries
return NewExecutor(m.pool, m.logger, progressUpdate).Execute(ctx, request, &filteredPolicy)
}
Expand Down
Loading