Skip to content

Commit ba9128a

Browse files
yevgenypatshermanschaaf
yevgenypats
and
hermanschaaf
authored
fix: Spec unmarshalling now supports defaults and validation (#181)
Fixes: defaults, validation and unknown keys Co-authored-by: Herman Schaaf <hermanschaaf@gmail.com>
1 parent 407b7e6 commit ba9128a

10 files changed

Lines changed: 332 additions & 168 deletions

File tree

‎specs/destination.go‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,6 @@ func (d *Destination) SetDefaults() {
3030
if d.Path == "" {
3131
d.Path = d.Name
3232
}
33-
if d.Version == "" {
34-
d.Version = "latest"
35-
}
3633
if d.Registry == RegistryGithub && !strings.Contains(d.Path, "/") {
3734
d.Path = "cloudquery/" + d.Path
3835
}
@@ -43,10 +40,23 @@ func (d *Destination) UnmarshalSpec(out interface{}) error {
4340
if err != nil {
4441
return err
4542
}
46-
dec := json.NewDecoder(nil)
43+
dec := json.NewDecoder(bytes.NewReader(b))
4744
dec.UseNumber()
4845
dec.DisallowUnknownFields()
49-
return json.Unmarshal(b, out)
46+
return dec.Decode(out)
47+
}
48+
49+
func (d *Destination) Validate() error {
50+
if d.Name == "" {
51+
return fmt.Errorf("name is required")
52+
}
53+
if d.Version == "" {
54+
return fmt.Errorf("version is required")
55+
}
56+
if !strings.HasPrefix(d.Version, "v") {
57+
return fmt.Errorf("version must start with v")
58+
}
59+
return nil
5060
}
5161

5262
func (m WriteMode) String() string {

‎specs/destination_test.go‎

Lines changed: 137 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package specs
22

33
import (
44
"testing"
5+
6+
"github.com/google/go-cmp/cmp"
57
)
68

79
type testDestinationSpec struct {
@@ -24,23 +26,7 @@ func TestWriteModeFromString(t *testing.T) {
2426
}
2527
}
2628

27-
func TestDestinationSetDefaults(t *testing.T) {
28-
destination := Destination{
29-
Name: "testDestination",
30-
}
31-
destination.SetDefaults()
32-
if destination.Registry != RegistryGithub {
33-
t.Fatalf("expected RegistryGithub, got %v", destination.Registry)
34-
}
35-
if destination.Path != "cloudquery/testDestination" {
36-
t.Fatalf("expected destination.Path (%s), got %s", destination.Name, destination.Path)
37-
}
38-
if destination.Version != "latest" {
39-
t.Fatalf("expected latest, got %s", destination.Version)
40-
}
41-
}
42-
43-
func TestDestinationUnmarshalSpec(t *testing.T) {
29+
func TestDestinationSpecUnmarshalSpec(t *testing.T) {
4430
destination := Destination{
4531
Spec: map[string]interface{}{
4632
"connection_string": "postgres://user:pass@host:port/db",
@@ -54,3 +40,137 @@ func TestDestinationUnmarshalSpec(t *testing.T) {
5440
t.Fatalf("expected postgres://user:pass@host:port/db, got %s", spec.ConnectionString)
5541
}
5642
}
43+
44+
var destinationUnmarshalSpecTestCases = []struct {
45+
name string
46+
spec string
47+
err string
48+
source *Source
49+
}{
50+
{
51+
"invalid_kind",
52+
`kind: nice`,
53+
"failed to decode spec: unknown kind nice",
54+
nil,
55+
},
56+
{
57+
"invalid_type",
58+
`kind: source
59+
spec:
60+
name: 3
61+
`,
62+
"failed to decode spec: json: cannot unmarshal number into Go struct field Source.name of type string",
63+
&Source{
64+
Name: "test",
65+
Tables: []string{"*"},
66+
},
67+
},
68+
{
69+
"unknown_field",
70+
`kind: source
71+
spec:
72+
namea: 3
73+
`,
74+
`failed to decode spec: json: unknown field "namea"`,
75+
&Source{
76+
Name: "test",
77+
Tables: []string{"*"},
78+
},
79+
},
80+
}
81+
82+
func TestDestinationUnmarshalSpec(t *testing.T) {
83+
for _, tc := range destinationUnmarshalSpecTestCases {
84+
t.Run(tc.name, func(t *testing.T) {
85+
var err error
86+
var spec Spec
87+
err = SpecUnmarshalYamlStrict([]byte(tc.spec), &spec)
88+
if err != nil {
89+
if err.Error() != tc.err {
90+
t.Fatalf("expected:%s got:%s", tc.err, err.Error())
91+
}
92+
return
93+
}
94+
95+
source := spec.Spec.(*Source)
96+
if cmp.Diff(source, tc.source) != "" {
97+
t.Fatalf("expected:%v got:%v", tc.source, source)
98+
}
99+
})
100+
}
101+
}
102+
103+
var destinationUnmarshalSpecValidateTestCases = []struct {
104+
name string
105+
spec string
106+
err string
107+
destination *Destination
108+
}{
109+
{
110+
"required_name",
111+
`kind: destination
112+
spec:`,
113+
"name is required",
114+
nil,
115+
},
116+
{
117+
"required_version",
118+
`kind: destination
119+
spec:
120+
name: test
121+
`,
122+
"version is required",
123+
nil,
124+
},
125+
{
126+
"required_version_format",
127+
`kind: destination
128+
spec:
129+
name: test
130+
version: 1.1.0
131+
`,
132+
"version must start with v",
133+
nil,
134+
},
135+
{
136+
"success",
137+
`kind: destination
138+
spec:
139+
name: test
140+
version: v1.1.0
141+
`,
142+
"",
143+
&Destination{
144+
Name: "test",
145+
Registry: RegistryGithub,
146+
Path: "cloudquery/test",
147+
Version: "v1.1.0",
148+
},
149+
},
150+
}
151+
152+
func TestDestinationUnmarshalSpecValidate(t *testing.T) {
153+
for _, tc := range destinationUnmarshalSpecValidateTestCases {
154+
t.Run(tc.name, func(t *testing.T) {
155+
var err error
156+
var spec Spec
157+
err = SpecUnmarshalYamlStrict([]byte(tc.spec), &spec)
158+
if err != nil {
159+
t.Fatal(err)
160+
}
161+
destination := spec.Spec.(*Destination)
162+
destination.SetDefaults()
163+
err = destination.Validate()
164+
if err != nil {
165+
if err.Error() != tc.err {
166+
t.Fatalf("expected:%s got:%s", tc.err, err.Error())
167+
}
168+
return
169+
}
170+
171+
if cmp.Diff(destination, tc.destination) != "" {
172+
t.Fatalf("expected:%v got:%v", tc.destination, destination)
173+
}
174+
})
175+
}
176+
}

‎specs/source.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package specs
22

33
import (
4+
"bytes"
45
"encoding/json"
6+
"fmt"
57
"strings"
6-
7-
"github.com/xeipuuv/gojsonschema"
88
)
99

1010
// Source is the spec for a source plugin
@@ -40,12 +40,12 @@ func (s *Source) SetDefaults() {
4040
if s.Path == "" {
4141
s.Path = s.Name
4242
}
43-
if s.Version == "" {
44-
s.Version = "latest"
45-
}
4643
if s.Registry == RegistryGithub && !strings.Contains(s.Path, "/") {
4744
s.Path = "cloudquery/" + s.Path
4845
}
46+
if s.Tables == nil {
47+
s.Tables = []string{"*"}
48+
}
4949
}
5050

5151
// UnmarshalSpec unmarshals the internal spec into the given interface
@@ -54,12 +54,24 @@ func (s *Source) UnmarshalSpec(out interface{}) error {
5454
if err != nil {
5555
return err
5656
}
57-
dec := json.NewDecoder(nil)
57+
dec := json.NewDecoder(bytes.NewReader(b))
5858
dec.UseNumber()
5959
dec.DisallowUnknownFields()
60-
return json.Unmarshal(b, out)
60+
return dec.Decode(out)
6161
}
6262

63-
func (*Source) Validate() (*gojsonschema.Result, error) {
64-
return nil, nil
63+
func (s *Source) Validate() error {
64+
if s.Name == "" {
65+
return fmt.Errorf("name is required")
66+
}
67+
if s.Version == "" {
68+
return fmt.Errorf("version is required")
69+
}
70+
if !strings.HasPrefix(s.Version, "v") {
71+
return fmt.Errorf("version must start with v")
72+
}
73+
if len(s.Destinations) == 0 {
74+
return fmt.Errorf("at least one destination is required")
75+
}
76+
return nil
6577
}

0 commit comments

Comments
 (0)