Skip to content

Commit 748cca7

Browse files
yevgenypatsyevgenypats
authored
feat: New file destination plugin (#6096)
Docs and tests passing.
1 parent 7bacdf3 commit 748cca7

35 files changed

Lines changed: 1556 additions & 0 deletions

‎.github/pr_labeler.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ sqlite:
4444
- plugins/destination/sqlite/**/*
4545
csv:
4646
- plugins/destination/csv/**/*
47+
file:
48+
- plugins/destination/file/**/*
4749
snowflake:
4850
- plugins/destination/snowflake/**/*
4951
bigquery:

‎.github/workflows/dest_file.yml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: Destination Plugin File Workflow
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "plugins/destination/filetypes/**"
7+
- "plugins/destination/file/**"
8+
- ".github/workflows/dest_file.yml"
9+
push:
10+
branches:
11+
- main
12+
paths:
13+
- "plugins/destination/filetypes/**"
14+
- "plugins/destination/file/**"
15+
- ".github/workflows/dest_file.yml"
16+
17+
jobs:
18+
plugins-destination-file:
19+
timeout-minutes: 30
20+
name: "plugins/destination/file"
21+
runs-on: ubuntu-latest
22+
defaults:
23+
run:
24+
working-directory: ./plugins/destination/file
25+
steps:
26+
- uses: actions/checkout@v3
27+
with:
28+
fetch-depth: 2
29+
- name: Set up Go 1.x
30+
uses: actions/setup-go@v3
31+
with:
32+
go-version-file: plugins/destination/file/go.mod
33+
cache: true
34+
cache-dependency-path: plugins/destination/file/go.sum
35+
- name: golangci-lint
36+
uses: cloudquery/golangci-lint-action@master
37+
with:
38+
version: v1.50.1
39+
working-directory: plugins/destination/file
40+
args: "--config ../../.golangci.yml"
41+
- name: Get dependencies
42+
run: go get -t -d ./...
43+
- name: Build
44+
run: go build .
45+
- name: Test file plugin
46+
run: make test
47+
validate-release:
48+
timeout-minutes: 30
49+
runs-on: ubuntu-latest
50+
env:
51+
CGO_ENABLED: 0
52+
steps:
53+
- name: Checkout
54+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
55+
uses: actions/checkout@v3
56+
- uses: actions/cache@v3
57+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
58+
with:
59+
path: |
60+
~/.cache/go-build
61+
~/go/pkg/mod
62+
key: ${{ runner.os }}-go-1.19.3-release-cache-${{ hashFiles('plugins/destination/file/go.sum') }}
63+
restore-keys: |
64+
${{ runner.os }}-go-1.19.3-release-cache-plugins-destination-file
65+
- name: Set up Go
66+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
67+
uses: actions/setup-go@v3
68+
with:
69+
go-version-file: plugins/destination/file/go.mod
70+
- name: Install GoReleaser
71+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
72+
uses: goreleaser/goreleaser-action@v3
73+
with:
74+
distribution: goreleaser-pro
75+
version: latest
76+
install-only: true
77+
- name: Run GoReleaser Dry-Run
78+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
79+
run: goreleaser release --snapshot --rm-dist --skip-validate --skip-publish --skip-sign -f ./plugins/destination/file/.goreleaser.yaml
80+
env:
81+
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

‎.github/workflows/wait_for_required_workflows.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ jobs:
6969
"plugins/destination/snowflake",
7070
"plugins/destination/bigquery",
7171
"plugins/destination/mongodb",
72+
"plugins/destination/file",
7273
].filter(action => matchesFile(action))
7374
if (actions.length === 0) {
7475
console.log("No actions to wait for")

‎plugins/destination/file/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
file

‎plugins/destination/file/.goreleaser.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
variables:
2+
component: destination/file
3+
binary: file
4+
5+
project_name: plugins/destination/file
6+
7+
monorepo:
8+
tag_prefix: plugins-destination-file-
9+
dir: plugins/destination/file
10+
11+
includes:
12+
- from_file:
13+
# Relative to the directory Go Releaser is run from (which is the root of the repository)
14+
path: ./plugins/.goreleaser.yaml

‎plugins/destination/file/CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Changelog
2+

‎plugins/destination/file/Makefile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
.PHONY: test
2+
test:
3+
go test -timeout 3m ./...
4+
5+
.PHONY: lint
6+
lint:
7+
@golangci-lint run --config ../../.golangci.yml

‎plugins/destination/file/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# CloudQuery File Destination Plugin
2+
3+
This destination plugin lets you sync data from a CloudQuery source to local files in various formats such as CSV and JSON.
4+
5+
## Links
6+
7+
- [User Guide](https://cloudquery.io/docs/plugins/destinations/file/overview)

‎plugins/destination/file/client/client.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package client
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
8+
"github.com/cloudquery/cloudquery/plugins/filetypes/csv"
9+
"github.com/cloudquery/cloudquery/plugins/filetypes/json"
10+
"github.com/cloudquery/plugin-sdk/plugins/destination"
11+
"github.com/cloudquery/plugin-sdk/specs"
12+
"github.com/rs/zerolog"
13+
)
14+
15+
type Client struct {
16+
destination.UnimplementedUnmanagedWriter
17+
logger zerolog.Logger
18+
spec specs.Destination
19+
pluginSpec Spec
20+
21+
csvTransformer *csv.Transformer
22+
csvReverseTransformer *csv.ReverseTransformer
23+
jsonTransformer *json.Transformer
24+
jsonReverseTransformer *json.ReverseTransformer
25+
}
26+
27+
func New(ctx context.Context, logger zerolog.Logger, spec specs.Destination) (destination.Client, error) {
28+
if spec.WriteMode != specs.WriteModeAppend {
29+
return nil, fmt.Errorf("file destination only supports append mode")
30+
}
31+
c := &Client{
32+
logger: logger.With().Str("module", "file").Logger(),
33+
spec: spec,
34+
csvTransformer: &csv.Transformer{},
35+
jsonTransformer: &json.Transformer{},
36+
csvReverseTransformer: &csv.ReverseTransformer{},
37+
jsonReverseTransformer: &json.ReverseTransformer{},
38+
}
39+
40+
if err := spec.UnmarshalSpec(&c.pluginSpec); err != nil {
41+
return nil, fmt.Errorf("failed to unmarshal postgresql spec: %w", err)
42+
}
43+
if err := c.pluginSpec.Validate(); err != nil {
44+
return nil, err
45+
}
46+
c.pluginSpec.SetDefaults()
47+
48+
if err := os.MkdirAll(c.pluginSpec.Directory, 0755); err != nil {
49+
return nil, fmt.Errorf("failed to create directory: %w", err)
50+
}
51+
52+
return c, nil
53+
}
54+
55+
func (*Client) Close(ctx context.Context) error {
56+
return nil
57+
}

‎plugins/destination/file/client/client_test.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package client
2+
3+
import (
4+
"testing"
5+
6+
"github.com/cloudquery/plugin-sdk/plugins/destination"
7+
)
8+
9+
func TestPluginCSV(t *testing.T) {
10+
p := destination.NewPlugin("file", "development", New, destination.WithManagedWriter())
11+
12+
destination.PluginTestSuiteRunner(t, p,
13+
Spec{
14+
Directory: t.TempDir(),
15+
Format: FormatTypeCSV,
16+
NoRotate: true,
17+
},
18+
destination.PluginTestSuiteTests{
19+
SkipOverwrite: true,
20+
SkipDeleteStale: true,
21+
},
22+
)
23+
}
24+
25+
func TestPluginJSON(t *testing.T) {
26+
p := destination.NewPlugin("file", "development", New, destination.WithManagedWriter())
27+
28+
destination.PluginTestSuiteRunner(t, p,
29+
Spec{
30+
Directory: t.TempDir(),
31+
Format: FormatTypeJSON,
32+
NoRotate: true,
33+
},
34+
destination.PluginTestSuiteTests{
35+
SkipOverwrite: true,
36+
SkipDeleteStale: true,
37+
},
38+
)
39+
}

0 commit comments

Comments
 (0)