Skip to content

Commit be2b696

Browse files
abochaboch
committed
Handle datastore update in Ipam and overlay drivers
Signed-off-by: Alessandro Boch <aboch@docker.com>
1 parent 48ebfd6 commit be2b696

File tree

10 files changed

+196
-77
lines changed

10 files changed

+196
-77
lines changed

‎datastore/datastore.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/docker/libkv/store/consul"
1414
"github.com/docker/libkv/store/etcd"
1515
"github.com/docker/libkv/store/zookeeper"
16+
"github.com/docker/libnetwork/discoverapi"
1617
"github.com/docker/libnetwork/types"
1718
)
1819

@@ -253,6 +254,29 @@ func NewDataStore(scope string, cfg *ScopeCfg) (DataStore, error) {
253254
return newClient(scope, cfg.Client.Provider, cfg.Client.Address, cfg.Client.Config, cached)
254255
}
255256

257+
// NewDataStoreFromConfig creates a new instance of LibKV data store starting from the datastore config data
258+
func NewDataStoreFromConfig(dsc discoverapi.DatastoreConfigData) (DataStore, error) {
259+
sCfg, ok := dsc.Config.(*store.Config)
260+
if !ok {
261+
return nil, fmt.Errorf("cannot parse store configuration: %v", dsc.Config)
262+
}
263+
264+
scopeCfg := &ScopeCfg{
265+
Client: ScopeClientCfg{
266+
Address: dsc.Address,
267+
Provider: dsc.Provider,
268+
Config: sCfg,
269+
},
270+
}
271+
272+
ds, err := NewDataStore(dsc.Scope, scopeCfg)
273+
if err != nil {
274+
return nil, fmt.Errorf("failed to construct datastore client from datastore configuration %v: %v", dsc, err)
275+
}
276+
277+
return ds, err
278+
}
279+
256280
func (ds *datastore) Close() {
257281
ds.store.Close()
258282
}

‎discoverapi/discoverapi.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ type DiscoveryType int
1616
const (
1717
// NodeDiscovery represents Node join/leave events provided by discovery
1818
NodeDiscovery = iota + 1
19-
// DatastoreUpdate represents a add/remove datastore event
20-
DatastoreUpdate
19+
// DatastoreConfig represents a add/remove datastore event
20+
DatastoreConfig
2121
)
2222

2323
// NodeDiscoveryData represents the structure backing the node discovery data json string
@@ -26,8 +26,9 @@ type NodeDiscoveryData struct {
2626
Self bool
2727
}
2828

29-
// DatastoreUpdateData is the data for the datastore update event message
30-
type DatastoreUpdateData struct {
29+
// DatastoreConfigData is the data for the datastore update event message
30+
type DatastoreConfigData struct {
31+
Scope string
3132
Provider string
3233
Address string
3334
Config interface{}

‎drivers.go‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package libnetwork
33
import (
44
"strings"
55

6+
"github.com/docker/libnetwork/discoverapi"
67
"github.com/docker/libnetwork/driverapi"
78
"github.com/docker/libnetwork/ipamapi"
89
builtinIpam "github.com/docker/libnetwork/ipams/builtin"
@@ -56,10 +57,12 @@ func makeDriverConfig(c *controller, ntype string) map[string]interface{} {
5657
if !v.IsValid() {
5758
continue
5859
}
59-
60-
config[netlabel.MakeKVProvider(k)] = v.Client.Provider
61-
config[netlabel.MakeKVProviderURL(k)] = v.Client.Address
62-
config[netlabel.MakeKVProviderConfig(k)] = v.Client.Config
60+
config[netlabel.MakeKVClient(k)] = discoverapi.DatastoreConfigData{
61+
Scope: k,
62+
Provider: v.Client.Provider,
63+
Address: v.Client.Address,
64+
Config: v.Client.Config,
65+
}
6366
}
6467

6568
return config

‎drivers/bridge/bridge_store.go‎

Lines changed: 8 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,37 +6,25 @@ import (
66
"net"
77

88
"github.com/Sirupsen/logrus"
9-
"github.com/docker/libkv/store"
109
"github.com/docker/libkv/store/boltdb"
1110
"github.com/docker/libnetwork/datastore"
11+
"github.com/docker/libnetwork/discoverapi"
1212
"github.com/docker/libnetwork/netlabel"
1313
"github.com/docker/libnetwork/types"
1414
)
1515

1616
const bridgePrefix = "bridge"
1717

1818
func (d *driver) initStore(option map[string]interface{}) error {
19-
var err error
20-
21-
provider, provOk := option[netlabel.LocalKVProvider]
22-
provURL, urlOk := option[netlabel.LocalKVProviderURL]
23-
24-
if provOk && urlOk {
25-
cfg := &datastore.ScopeCfg{
26-
Client: datastore.ScopeClientCfg{
27-
Provider: provider.(string),
28-
Address: provURL.(string),
29-
},
19+
if data, ok := option[netlabel.LocalKVClient]; ok {
20+
var err error
21+
dsc, ok := data.(discoverapi.DatastoreConfigData)
22+
if !ok {
23+
return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
3024
}
31-
32-
provConfig, confOk := option[netlabel.LocalKVProviderConfig]
33-
if confOk {
34-
cfg.Client.Config = provConfig.(*store.Config)
35-
}
36-
37-
d.store, err = datastore.NewDataStore(datastore.LocalScope, cfg)
25+
d.store, err = datastore.NewDataStoreFromConfig(dsc)
3826
if err != nil {
39-
return fmt.Errorf("bridge driver failed to initialize data store: %v", err)
27+
return types.InternalErrorf("bridge driver failed to initialize data store: %v", err)
4028
}
4129

4230
return d.populateNetworks()

‎drivers/overlay/overlay.go‎

Lines changed: 52 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@ import (
66
"sync"
77

88
"github.com/Sirupsen/logrus"
9-
"github.com/docker/libkv/store"
109
"github.com/docker/libnetwork/datastore"
1110
"github.com/docker/libnetwork/discoverapi"
1211
"github.com/docker/libnetwork/driverapi"
1312
"github.com/docker/libnetwork/idm"
1413
"github.com/docker/libnetwork/netlabel"
14+
"github.com/docker/libnetwork/types"
1515
"github.com/hashicorp/serf/serf"
1616
)
1717

@@ -25,6 +25,8 @@ const (
2525
vxlanVethMTU = 1450
2626
)
2727

28+
var initVxlanIdm = make(chan (bool), 1)
29+
2830
type driver struct {
2931
eventCh chan serf.Event
3032
notifyCh chan ovNotify
@@ -56,6 +58,18 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
5658
config: config,
5759
}
5860

61+
if data, ok := config[netlabel.GlobalKVClient]; ok {
62+
var err error
63+
dsc, ok := data.(discoverapi.DatastoreConfigData)
64+
if !ok {
65+
return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
66+
}
67+
d.store, err = datastore.NewDataStoreFromConfig(dsc)
68+
if err != nil {
69+
return types.InternalErrorf("failed to initialize data store: %v", err)
70+
}
71+
}
72+
5973
return dc.RegisterDriver(networkType, d, c)
6074
}
6175

@@ -73,42 +87,33 @@ func Fini(drv driverapi.Driver) {
7387
}
7488

7589
func (d *driver) configure() error {
76-
var err error
90+
if d.store == nil {
91+
return types.NoServiceErrorf("datastore is not available")
92+
}
7793

78-
if len(d.config) == 0 {
79-
return nil
94+
if d.vxlanIdm == nil {
95+
return d.initializeVxlanIdm()
8096
}
8197

82-
d.once.Do(func() {
83-
provider, provOk := d.config[netlabel.GlobalKVProvider]
84-
provURL, urlOk := d.config[netlabel.GlobalKVProviderURL]
98+
return nil
99+
}
85100

86-
if provOk && urlOk {
87-
cfg := &datastore.ScopeCfg{
88-
Client: datastore.ScopeClientCfg{
89-
Provider: provider.(string),
90-
Address: provURL.(string),
91-
},
92-
}
93-
provConfig, confOk := d.config[netlabel.GlobalKVProviderConfig]
94-
if confOk {
95-
cfg.Client.Config = provConfig.(*store.Config)
96-
}
97-
d.store, err = datastore.NewDataStore(datastore.GlobalScope, cfg)
98-
if err != nil {
99-
err = fmt.Errorf("failed to initialize data store: %v", err)
100-
return
101-
}
102-
}
101+
func (d *driver) initializeVxlanIdm() error {
102+
var err error
103103

104-
d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd)
105-
if err != nil {
106-
err = fmt.Errorf("failed to initialize vxlan id manager: %v", err)
107-
return
108-
}
109-
})
104+
initVxlanIdm <- true
105+
defer func() { <-initVxlanIdm }()
106+
107+
if d.vxlanIdm != nil {
108+
return nil
109+
}
110+
111+
d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd)
112+
if err != nil {
113+
return fmt.Errorf("failed to initialize vxlan id manager: %v", err)
114+
}
110115

111-
return err
116+
return nil
112117
}
113118

114119
func (d *driver) Type() string {
@@ -187,12 +192,27 @@ func (d *driver) pushLocalEndpointEvent(action, nid, eid string) {
187192

188193
// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
189194
func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
190-
if dType == discoverapi.NodeDiscovery {
195+
switch dType {
196+
case discoverapi.NodeDiscovery:
191197
nodeData, ok := data.(discoverapi.NodeDiscoveryData)
192198
if !ok || nodeData.Address == "" {
193199
return fmt.Errorf("invalid discovery data")
194200
}
195201
d.nodeJoin(nodeData.Address, nodeData.Self)
202+
case discoverapi.DatastoreConfig:
203+
var err error
204+
if d.store != nil {
205+
return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already")
206+
}
207+
dsc, ok := data.(discoverapi.DatastoreConfigData)
208+
if !ok {
209+
return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
210+
}
211+
d.store, err = datastore.NewDataStoreFromConfig(dsc)
212+
if err != nil {
213+
return types.InternalErrorf("failed to initialize data store: %v", err)
214+
}
215+
default:
196216
}
197217
return nil
198218
}

‎drivers/overlay/overlay_test.go‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/docker/libnetwork/discoverapi"
99
"github.com/docker/libnetwork/driverapi"
1010
_ "github.com/docker/libnetwork/testutils"
11+
"github.com/docker/libnetwork/types"
1112
)
1213

1314
type driverTester struct {
@@ -23,8 +24,12 @@ func setupDriver(t *testing.T) *driverTester {
2324
t.Fatal(err)
2425
}
2526

26-
if err := dt.d.configure(); err != nil {
27-
t.Fatal(err)
27+
err := dt.d.configure()
28+
if err == nil {
29+
t.Fatalf("Failed to detect nil store")
30+
}
31+
if _, ok := err.(types.NoServiceError); !ok {
32+
t.Fatalf("Unexpected error type: %v", err)
2833
}
2934

3035
iface, err := net.InterfaceByName("eth0")
@@ -94,8 +99,12 @@ func TestOverlayNilConfig(t *testing.T) {
9499
t.Fatal(err)
95100
}
96101

97-
if err := dt.d.configure(); err != nil {
98-
t.Fatal(err)
102+
err := dt.d.configure()
103+
if err == nil {
104+
t.Fatalf("Failed to detect nil store")
105+
}
106+
if _, ok := err.(types.NoServiceError); !ok {
107+
t.Fatalf("Unexpected error type: %v", err)
99108
}
100109

101110
cleanupDriver(t, dt)

0 commit comments

Comments
 (0)