diff --git a/api/ca.pb.go b/api/ca.pb.go index 3262fe4f66..19d103e84e 100644 --- a/api/ca.pb.go +++ b/api/ca.pb.go @@ -89,6 +89,22 @@ func (m *GetRootCACertificateResponse) Reset() { *m = GetRoot func (*GetRootCACertificateResponse) ProtoMessage() {} func (*GetRootCACertificateResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{5} } +type GetUnlockKeyRequest struct { +} + +func (m *GetUnlockKeyRequest) Reset() { *m = GetUnlockKeyRequest{} } +func (*GetUnlockKeyRequest) ProtoMessage() {} +func (*GetUnlockKeyRequest) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{6} } + +type GetUnlockKeyResponse struct { + UnlockKey []byte `protobuf:"bytes,1,opt,name=unlock_key,json=unlockKey,proto3" json:"unlock_key,omitempty"` + Version Version `protobuf:"bytes,2,opt,name=version" json:"version"` +} + +func (m *GetUnlockKeyResponse) Reset() { *m = GetUnlockKeyResponse{} } +func (*GetUnlockKeyResponse) ProtoMessage() {} +func (*GetUnlockKeyResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{7} } + func init() { proto.RegisterType((*NodeCertificateStatusRequest)(nil), "docker.swarmkit.v1.NodeCertificateStatusRequest") proto.RegisterType((*NodeCertificateStatusResponse)(nil), "docker.swarmkit.v1.NodeCertificateStatusResponse") @@ -96,6 +112,8 @@ func init() { proto.RegisterType((*IssueNodeCertificateResponse)(nil), "docker.swarmkit.v1.IssueNodeCertificateResponse") proto.RegisterType((*GetRootCACertificateRequest)(nil), "docker.swarmkit.v1.GetRootCACertificateRequest") proto.RegisterType((*GetRootCACertificateResponse)(nil), "docker.swarmkit.v1.GetRootCACertificateResponse") + proto.RegisterType((*GetUnlockKeyRequest)(nil), "docker.swarmkit.v1.GetUnlockKeyRequest") + proto.RegisterType((*GetUnlockKeyResponse)(nil), "docker.swarmkit.v1.GetUnlockKeyResponse") } type authenticatedWrapperCAServer struct { @@ -115,6 +133,14 @@ func (p *authenticatedWrapperCAServer) GetRootCACertificate(ctx context.Context, return p.local.GetRootCACertificate(ctx, r) } +func (p *authenticatedWrapperCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) { + + if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil { + return nil, err + } + return p.local.GetUnlockKey(ctx, r) +} + type authenticatedWrapperNodeCAServer struct { local NodeCAServer authorize func(context.Context, []string) error @@ -211,6 +237,29 @@ func (m *GetRootCACertificateResponse) Copy() *GetRootCACertificateResponse { return o } +func (m *GetUnlockKeyRequest) Copy() *GetUnlockKeyRequest { + if m == nil { + return nil + } + + o := &GetUnlockKeyRequest{} + + return o +} + +func (m *GetUnlockKeyResponse) Copy() *GetUnlockKeyResponse { + if m == nil { + return nil + } + + o := &GetUnlockKeyResponse{ + UnlockKey: m.UnlockKey, + Version: *m.Version.Copy(), + } + + return o +} + func (this *NodeCertificateStatusRequest) GoString() string { if this == nil { return "nil" @@ -278,6 +327,26 @@ func (this *GetRootCACertificateResponse) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *GetUnlockKeyRequest) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 4) + s = append(s, "&api.GetUnlockKeyRequest{") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *GetUnlockKeyResponse) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&api.GetUnlockKeyResponse{") + s = append(s, "UnlockKey: "+fmt.Sprintf("%#v", this.UnlockKey)+",\n") + s = append(s, "Version: "+strings.Replace(this.Version.GoString(), `&`, ``, 1)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func valueToGoStringCa(v interface{}, typ string) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -317,6 +386,9 @@ const _ = grpc.SupportPackageIsVersion3 type CAClient interface { GetRootCACertificate(ctx context.Context, in *GetRootCACertificateRequest, opts ...grpc.CallOption) (*GetRootCACertificateResponse, error) + // GetUnlockKey returns the current unlock key for the cluster for the role of the client + // asking. + GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error) } type cAClient struct { @@ -336,10 +408,22 @@ func (c *cAClient) GetRootCACertificate(ctx context.Context, in *GetRootCACertif return out, nil } +func (c *cAClient) GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error) { + out := new(GetUnlockKeyResponse) + err := grpc.Invoke(ctx, "/docker.swarmkit.v1.CA/GetUnlockKey", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for CA service type CAServer interface { GetRootCACertificate(context.Context, *GetRootCACertificateRequest) (*GetRootCACertificateResponse, error) + // GetUnlockKey returns the current unlock key for the cluster for the role of the client + // asking. + GetUnlockKey(context.Context, *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) } func RegisterCAServer(s *grpc.Server, srv CAServer) { @@ -364,6 +448,24 @@ func _CA_GetRootCACertificate_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _CA_GetUnlockKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetUnlockKeyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CAServer).GetUnlockKey(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/docker.swarmkit.v1.CA/GetUnlockKey", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CAServer).GetUnlockKey(ctx, req.(*GetUnlockKeyRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _CA_serviceDesc = grpc.ServiceDesc{ ServiceName: "docker.swarmkit.v1.CA", HandlerType: (*CAServer)(nil), @@ -372,6 +474,10 @@ var _CA_serviceDesc = grpc.ServiceDesc{ MethodName: "GetRootCACertificate", Handler: _CA_GetRootCACertificate_Handler, }, + { + MethodName: "GetUnlockKey", + Handler: _CA_GetUnlockKey_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: fileDescriptorCa, @@ -642,6 +748,56 @@ func (m *GetRootCACertificateResponse) MarshalTo(data []byte) (int, error) { return i, nil } +func (m *GetUnlockKeyRequest) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *GetUnlockKeyRequest) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + +func (m *GetUnlockKeyResponse) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *GetUnlockKeyResponse) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.UnlockKey) > 0 { + data[i] = 0xa + i++ + i = encodeVarintCa(data, i, uint64(len(m.UnlockKey))) + i += copy(data[i:], m.UnlockKey) + } + data[i] = 0x12 + i++ + i = encodeVarintCa(data, i, uint64(m.Version.Size())) + n3, err := m.Version.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n3 + return i, nil +} + func encodeFixed64Ca(data []byte, offset int, v uint64) int { data[offset] = uint8(v) data[offset+1] = uint8(v >> 8) @@ -767,6 +923,37 @@ func (p *raftProxyCAServer) GetRootCACertificate(ctx context.Context, r *GetRoot return resp, err } +func (p *raftProxyCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) { + + conn, err := p.connSelector.LeaderConn(ctx) + if err != nil { + if err == raftselector.ErrIsLeader { + return p.local.GetUnlockKey(ctx, r) + } + return nil, err + } + modCtx, err := p.runCtxMods(ctx) + if err != nil { + return nil, err + } + + resp, err := NewCAClient(conn).GetUnlockKey(modCtx, r) + if err != nil { + if !strings.Contains(err.Error(), "is closing") && !strings.Contains(err.Error(), "the connection is unavailable") && !strings.Contains(err.Error(), "connection error") { + return resp, err + } + conn, err := p.pollNewLeaderConn(ctx) + if err != nil { + if err == raftselector.ErrIsLeader { + return p.local.GetUnlockKey(ctx, r) + } + return nil, err + } + return NewCAClient(conn).GetUnlockKey(modCtx, r) + } + return resp, err +} + type raftProxyNodeCAServer struct { local NodeCAServer connSelector raftselector.ConnProvider @@ -965,6 +1152,24 @@ func (m *GetRootCACertificateResponse) Size() (n int) { return n } +func (m *GetUnlockKeyRequest) Size() (n int) { + var l int + _ = l + return n +} + +func (m *GetUnlockKeyResponse) Size() (n int) { + var l int + _ = l + l = len(m.UnlockKey) + if l > 0 { + n += 1 + l + sovCa(uint64(l)) + } + l = m.Version.Size() + n += 1 + l + sovCa(uint64(l)) + return n +} + func sovCa(x uint64) (n int) { for { n++ @@ -1041,6 +1246,26 @@ func (this *GetRootCACertificateResponse) String() string { }, "") return s } +func (this *GetUnlockKeyRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&GetUnlockKeyRequest{`, + `}`, + }, "") + return s +} +func (this *GetUnlockKeyResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&GetUnlockKeyResponse{`, + `UnlockKey:` + fmt.Sprintf("%v", this.UnlockKey) + `,`, + `Version:` + strings.Replace(strings.Replace(this.Version.String(), "Version", "Version", 1), `&`, ``, 1) + `,`, + `}`, + }, "") + return s +} func valueToStringCa(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -1602,6 +1827,167 @@ func (m *GetRootCACertificateResponse) Unmarshal(data []byte) error { } return nil } +func (m *GetUnlockKeyRequest) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetUnlockKeyRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetUnlockKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipCa(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCa + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GetUnlockKeyResponse) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetUnlockKeyResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetUnlockKeyResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UnlockKey", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthCa + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UnlockKey = append(m.UnlockKey[:0], data[iNdEx:postIndex]...) + if m.UnlockKey == nil { + m.UnlockKey = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCa + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCa(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCa + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipCa(data []byte) (n int, err error) { l := len(data) iNdEx := 0 @@ -1710,36 +2096,42 @@ var ( func init() { proto.RegisterFile("ca.proto", fileDescriptorCa) } var fileDescriptorCa = []byte{ - // 493 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x94, 0xcf, 0x6e, 0xd3, 0x40, - 0x10, 0xc6, 0xbb, 0x0e, 0xa4, 0x65, 0x52, 0x05, 0xb4, 0x04, 0x29, 0xa4, 0xa9, 0x53, 0x99, 0x03, - 0x9c, 0x9c, 0xd6, 0x70, 0xe2, 0x44, 0x62, 0x24, 0x94, 0x03, 0x08, 0x6d, 0x1e, 0x00, 0xb9, 0xf6, - 0x10, 0xac, 0x24, 0x5e, 0xe3, 0xdd, 0x80, 0xb8, 0x21, 0x81, 0x38, 0x70, 0x47, 0x70, 0xe2, 0x11, - 0x78, 0x8e, 0x8a, 0x13, 0x47, 0x4e, 0x15, 0xf1, 0x03, 0x20, 0x1e, 0x01, 0xed, 0xda, 0x21, 0xfd, - 0xb3, 0x89, 0xca, 0xc9, 0x3b, 0xb3, 0xf3, 0x7d, 0xfe, 0xed, 0x8c, 0xd7, 0xb0, 0x15, 0x06, 0x6e, - 0x9a, 0x71, 0xc9, 0x29, 0x8d, 0x78, 0x38, 0xc6, 0xcc, 0x15, 0xaf, 0x83, 0x6c, 0x3a, 0x8e, 0xa5, - 0xfb, 0xea, 0xa0, 0x55, 0x93, 0x6f, 0x52, 0x14, 0x45, 0x41, 0xab, 0x26, 0x52, 0x0c, 0x17, 0x41, - 0x63, 0xc4, 0x47, 0x5c, 0x2f, 0xbb, 0x6a, 0x55, 0x66, 0xaf, 0xa7, 0x93, 0xd9, 0x28, 0x4e, 0xba, - 0xc5, 0xa3, 0x48, 0x3a, 0x3e, 0xb4, 0x9f, 0xf0, 0x08, 0x7d, 0xcc, 0x64, 0xfc, 0x3c, 0x0e, 0x03, - 0x89, 0x43, 0x19, 0xc8, 0x99, 0x60, 0xf8, 0x72, 0x86, 0x42, 0xd2, 0x5b, 0xb0, 0x99, 0xf0, 0x08, - 0x9f, 0xc5, 0x51, 0x93, 0xec, 0x91, 0x3b, 0x57, 0xfa, 0x90, 0x1f, 0x77, 0xaa, 0x4a, 0x32, 0x78, - 0xc8, 0xaa, 0x6a, 0x6b, 0x10, 0x39, 0x5f, 0x09, 0xec, 0xae, 0x70, 0x11, 0x29, 0x4f, 0x04, 0xd2, - 0xfb, 0x50, 0x15, 0x3a, 0xa3, 0x5d, 0x6a, 0x9e, 0xe3, 0x9e, 0x3f, 0x90, 0x3b, 0x10, 0x62, 0x16, - 0x24, 0xe1, 0x42, 0x5b, 0x2a, 0x68, 0x0f, 0x6a, 0xe1, 0xd2, 0xb8, 0x69, 0x69, 0x83, 0x8e, 0xc9, - 0xe0, 0xc4, 0xfb, 0xd9, 0x49, 0x8d, 0xf3, 0x9e, 0xc0, 0x8e, 0x72, 0xc7, 0x33, 0x94, 0x8b, 0x53, - 0xde, 0x83, 0x4b, 0x19, 0x9f, 0xa0, 0x86, 0xab, 0x7b, 0x6d, 0x93, 0xb7, 0x52, 0x32, 0x3e, 0xc1, - 0xbe, 0xd5, 0x24, 0x4c, 0x57, 0xd3, 0x9b, 0x50, 0x09, 0x45, 0xa6, 0x81, 0xb6, 0xfb, 0x9b, 0xf9, - 0x71, 0xa7, 0xe2, 0x0f, 0x19, 0x53, 0x39, 0xda, 0x80, 0xcb, 0x92, 0x8f, 0x31, 0x69, 0x56, 0x54, - 0xd3, 0x58, 0x11, 0x38, 0x9f, 0x08, 0xb4, 0xcd, 0x18, 0x65, 0x9b, 0x2e, 0xd2, 0x6d, 0xfa, 0x14, - 0xae, 0xea, 0xa2, 0x29, 0x4e, 0x0f, 0x31, 0x13, 0x2f, 0xe2, 0x54, 0x23, 0xd4, 0xbd, 0xdb, 0xab, - 0xb8, 0x87, 0x29, 0x86, 0xee, 0xe3, 0x7f, 0xe5, 0xac, 0xae, 0xf4, 0xcb, 0xd8, 0xd9, 0x85, 0x9d, - 0x47, 0x28, 0x19, 0xe7, 0xd2, 0xef, 0x9d, 0xef, 0x8e, 0xf3, 0x00, 0xda, 0xe6, 0xed, 0x92, 0x7a, - 0xef, 0xf4, 0x80, 0x14, 0xf9, 0xf6, 0xa9, 0xfe, 0x7b, 0x1f, 0x09, 0x58, 0x7e, 0x8f, 0xbe, 0x23, - 0xd0, 0x30, 0x39, 0xd1, 0xae, 0x89, 0x7c, 0x0d, 0x52, 0x6b, 0xff, 0xe2, 0x82, 0x02, 0xd2, 0xd9, - 0xfa, 0xfe, 0xed, 0xf7, 0x17, 0xcb, 0xba, 0x46, 0xbc, 0xcf, 0x16, 0xe8, 0x96, 0x96, 0x40, 0xa6, - 0x81, 0x98, 0x81, 0xd6, 0x7c, 0x41, 0x66, 0xa0, 0x75, 0xb3, 0x5e, 0x02, 0xd1, 0x0f, 0x04, 0x6e, - 0x18, 0xaf, 0x0f, 0xdd, 0x5f, 0x35, 0xd1, 0x55, 0xf7, 0xb5, 0x75, 0xf0, 0x1f, 0x8a, 0xb3, 0x20, - 0xfd, 0xf6, 0xd1, 0xdc, 0xde, 0xf8, 0x39, 0xb7, 0x37, 0xfe, 0xcc, 0x6d, 0xf2, 0x36, 0xb7, 0xc9, - 0x51, 0x6e, 0x93, 0x1f, 0xb9, 0x4d, 0x7e, 0xe5, 0x36, 0x39, 0xac, 0xea, 0x3f, 0xc6, 0xdd, 0xbf, - 0x01, 0x00, 0x00, 0xff, 0xff, 0xb3, 0xf8, 0x41, 0xef, 0x96, 0x04, 0x00, 0x00, + // 586 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcb, 0x6e, 0xd3, 0x40, + 0x14, 0xcd, 0x38, 0x25, 0x69, 0x6f, 0x42, 0x8a, 0xa6, 0x89, 0x14, 0xf2, 0x70, 0x2a, 0xb3, 0x68, + 0x37, 0x38, 0x6d, 0x60, 0x05, 0x1b, 0x92, 0x20, 0x55, 0x11, 0x02, 0x21, 0x47, 0xb0, 0xad, 0x5c, + 0x67, 0x08, 0x56, 0x12, 0x8f, 0xf1, 0x8c, 0x0b, 0xd9, 0x21, 0x51, 0xf1, 0x07, 0x08, 0x56, 0x7c, + 0x02, 0xdf, 0x11, 0xb1, 0x62, 0xc9, 0x2a, 0x22, 0xfe, 0x00, 0xc4, 0x27, 0x20, 0x8f, 0x6d, 0x9a, + 0x87, 0x13, 0xda, 0x55, 0x3c, 0xd7, 0xe7, 0x9c, 0x7b, 0xee, 0xc9, 0xf5, 0xc0, 0xb6, 0xa1, 0xab, + 0xb6, 0x43, 0x39, 0xc5, 0xb8, 0x47, 0x8d, 0x01, 0x71, 0x54, 0xf6, 0x56, 0x77, 0x46, 0x03, 0x93, + 0xab, 0xe7, 0xc7, 0xa5, 0x0c, 0x1f, 0xdb, 0x84, 0x05, 0x80, 0x52, 0x86, 0xd9, 0xc4, 0x88, 0x0e, + 0xf9, 0x3e, 0xed, 0x53, 0xf1, 0x58, 0xf7, 0x9f, 0xc2, 0xea, 0x9e, 0x3d, 0x74, 0xfb, 0xa6, 0x55, + 0x0f, 0x7e, 0x82, 0xa2, 0xd2, 0x86, 0xca, 0x33, 0xda, 0x23, 0x6d, 0xe2, 0x70, 0xf3, 0x95, 0x69, + 0xe8, 0x9c, 0x74, 0xb9, 0xce, 0x5d, 0xa6, 0x91, 0x37, 0x2e, 0x61, 0x1c, 0xdf, 0x81, 0xb4, 0x45, + 0x7b, 0xe4, 0xd4, 0xec, 0x15, 0xd1, 0x3e, 0x3a, 0xdc, 0x69, 0x81, 0x37, 0xad, 0xa5, 0x7c, 0x4a, + 0xe7, 0xb1, 0x96, 0xf2, 0x5f, 0x75, 0x7a, 0xca, 0x57, 0x04, 0xd5, 0x35, 0x2a, 0xcc, 0xa6, 0x16, + 0x23, 0xf8, 0x01, 0xa4, 0x98, 0xa8, 0x08, 0x95, 0x4c, 0x43, 0x51, 0x57, 0x07, 0x52, 0x3b, 0x8c, + 0xb9, 0xba, 0x65, 0x44, 0xdc, 0x90, 0x81, 0x9b, 0x90, 0x31, 0x2e, 0x85, 0x8b, 0x92, 0x10, 0xa8, + 0xc5, 0x09, 0xcc, 0xf5, 0xd7, 0xe6, 0x39, 0xca, 0x05, 0x82, 0xb2, 0xaf, 0x4e, 0x96, 0x5c, 0x46, + 0x53, 0xde, 0x87, 0x2d, 0x87, 0x0e, 0x89, 0x30, 0x97, 0x6b, 0x54, 0xe2, 0xb4, 0x7d, 0xa6, 0x46, + 0x87, 0xa4, 0x25, 0x15, 0x91, 0x26, 0xd0, 0xf8, 0x36, 0x24, 0x0d, 0xe6, 0x08, 0x43, 0xd9, 0x56, + 0xda, 0x9b, 0xd6, 0x92, 0xed, 0xae, 0xa6, 0xf9, 0x35, 0x9c, 0x87, 0x1b, 0x9c, 0x0e, 0x88, 0x55, + 0x4c, 0xfa, 0xa1, 0x69, 0xc1, 0x41, 0xf9, 0x84, 0xa0, 0x12, 0x6f, 0x23, 0x8c, 0xe9, 0x2a, 0x69, + 0xe3, 0xe7, 0xb0, 0x2b, 0x40, 0x23, 0x32, 0x3a, 0x23, 0x0e, 0x7b, 0x6d, 0xda, 0xc2, 0x42, 0xae, + 0x71, 0xb0, 0xce, 0x77, 0xd7, 0x26, 0x86, 0xfa, 0xf4, 0x1f, 0x5c, 0xcb, 0xf9, 0xfc, 0xcb, 0xb3, + 0x52, 0x85, 0xf2, 0x09, 0xe1, 0x1a, 0xa5, 0xbc, 0xdd, 0x5c, 0x4d, 0x47, 0x79, 0x04, 0x95, 0xf8, + 0xd7, 0xa1, 0xeb, 0xfd, 0xc5, 0x3f, 0xc8, 0x77, 0x9e, 0x5d, 0xcc, 0xbf, 0x00, 0x7b, 0x27, 0x84, + 0xbf, 0xb0, 0x86, 0xd4, 0x18, 0x3c, 0x21, 0xe3, 0x48, 0xd8, 0x81, 0xfc, 0x62, 0x39, 0x14, 0xac, + 0x02, 0xb8, 0xa2, 0x78, 0x3a, 0x20, 0xe3, 0x50, 0x6f, 0xc7, 0x8d, 0x60, 0xf8, 0x21, 0xa4, 0xcf, + 0x89, 0xc3, 0x4c, 0x6a, 0x85, 0xcb, 0x50, 0x8e, 0x1b, 0xfc, 0x65, 0x00, 0x69, 0x6d, 0x4d, 0xa6, + 0xb5, 0x84, 0x16, 0x31, 0x1a, 0x17, 0x12, 0x48, 0xed, 0x26, 0xfe, 0x80, 0x44, 0xef, 0x95, 0xa1, + 0x70, 0x3d, 0x4e, 0x6b, 0x43, 0x3a, 0xa5, 0xa3, 0xab, 0x13, 0x82, 0xf1, 0x94, 0xed, 0xef, 0xdf, + 0x7e, 0x7f, 0x91, 0xa4, 0x5b, 0x08, 0xbf, 0x83, 0xec, 0x7c, 0x00, 0xf8, 0x60, 0x8d, 0xd6, 0x72, + 0x72, 0xa5, 0xc3, 0xff, 0x03, 0xc3, 0x66, 0x05, 0xd1, 0x6c, 0x17, 0x6e, 0x0a, 0xe4, 0xdd, 0x91, + 0x6e, 0xe9, 0x7d, 0xe2, 0x34, 0x3e, 0x4b, 0x20, 0xf6, 0x2a, 0x8c, 0x22, 0x6e, 0x2b, 0xe3, 0xa3, + 0xd8, 0xf0, 0x19, 0xc5, 0x47, 0xb1, 0x69, 0xe1, 0xe7, 0xa2, 0xf8, 0x88, 0xa0, 0x10, 0x7b, 0x87, + 0xe0, 0xa3, 0x75, 0x6b, 0xbd, 0xee, 0xd2, 0x2a, 0x1d, 0x5f, 0x83, 0xb1, 0x6c, 0xa4, 0x55, 0x99, + 0xcc, 0xe4, 0xc4, 0xcf, 0x99, 0x9c, 0xf8, 0x33, 0x93, 0xd1, 0x7b, 0x4f, 0x46, 0x13, 0x4f, 0x46, + 0x3f, 0x3c, 0x19, 0xfd, 0xf2, 0x64, 0x74, 0x96, 0x12, 0xd7, 0xe6, 0xbd, 0xbf, 0x01, 0x00, 0x00, + 0xff, 0xff, 0xe7, 0x80, 0x3b, 0x00, 0x9b, 0x05, 0x00, 0x00, } diff --git a/api/ca.proto b/api/ca.proto index 2bfa9f7f81..5aa1f673ee 100644 --- a/api/ca.proto +++ b/api/ca.proto @@ -13,6 +13,11 @@ service CA { rpc GetRootCACertificate(GetRootCACertificateRequest) returns (GetRootCACertificateResponse) { option (docker.protobuf.plugin.tls_authorization) = { insecure: true }; }; + // GetUnlockKey returns the current unlock key for the cluster for the role of the client + // asking. + rpc GetUnlockKey(GetUnlockKeyRequest) returns (GetUnlockKeyResponse) { + option (docker.protobuf.plugin.tls_authorization) = { roles: ["swarm-manager"] }; + }; } service NodeCA { @@ -55,3 +60,10 @@ message GetRootCACertificateRequest {} message GetRootCACertificateResponse { bytes certificate = 1; } + +message GetUnlockKeyRequest {} + +message GetUnlockKeyResponse { + bytes unlock_key = 1; + Version version = 2 [(gogoproto.nullable) = false]; +} diff --git a/api/control.pb.go b/api/control.pb.go index 0cff716d22..6f36208c2c 100644 --- a/api/control.pb.go +++ b/api/control.pb.go @@ -405,16 +405,19 @@ func (m *ListClustersResponse) Reset() { *m = ListClustersRes func (*ListClustersResponse) ProtoMessage() {} func (*ListClustersResponse) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{35} } -type JoinTokenRotation struct { - // RotateWorkerToken tells UpdateCluster to rotate the worker secret. - RotateWorkerToken bool `protobuf:"varint,1,opt,name=rotate_worker_token,json=rotateWorkerToken,proto3" json:"rotate_worker_token,omitempty"` - // RotateManagerSecret tells UpdateCluster to rotate the manager secret. - RotateManagerToken bool `protobuf:"varint,2,opt,name=rotate_manager_token,json=rotateManagerToken,proto3" json:"rotate_manager_token,omitempty"` +// KeyRotation tells UpdateCluster what items to rotate +type KeyRotation struct { + // WorkerJoinToken tells UpdateCluster to rotate the worker secret token. + WorkerJoinToken bool `protobuf:"varint,1,opt,name=worker_join_token,json=workerJoinToken,proto3" json:"worker_join_token,omitempty"` + // ManagerJoinToken tells UpdateCluster to rotate the manager secret token. + ManagerJoinToken bool `protobuf:"varint,2,opt,name=manager_join_token,json=managerJoinToken,proto3" json:"manager_join_token,omitempty"` + // ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key + ManagerUnlockKey bool `protobuf:"varint,3,opt,name=manager_unlock_key,json=managerUnlockKey,proto3" json:"manager_unlock_key,omitempty"` } -func (m *JoinTokenRotation) Reset() { *m = JoinTokenRotation{} } -func (*JoinTokenRotation) ProtoMessage() {} -func (*JoinTokenRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} } +func (m *KeyRotation) Reset() { *m = KeyRotation{} } +func (*KeyRotation) ProtoMessage() {} +func (*KeyRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} } type UpdateClusterRequest struct { // ClusterID is the cluster ID to update. @@ -423,8 +426,8 @@ type UpdateClusterRequest struct { ClusterVersion *Version `protobuf:"bytes,2,opt,name=cluster_version,json=clusterVersion" json:"cluster_version,omitempty"` // Spec is the new spec to apply to the cluster. Spec *ClusterSpec `protobuf:"bytes,3,opt,name=spec" json:"spec,omitempty"` - // Rotation contains flags for join token rotation - Rotation JoinTokenRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"` + // Rotation contains flags for join token and unlock key rotation + Rotation KeyRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"` } func (m *UpdateClusterRequest) Reset() { *m = UpdateClusterRequest{} } @@ -598,7 +601,7 @@ func init() { proto.RegisterType((*ListClustersRequest)(nil), "docker.swarmkit.v1.ListClustersRequest") proto.RegisterType((*ListClustersRequest_Filters)(nil), "docker.swarmkit.v1.ListClustersRequest.Filters") proto.RegisterType((*ListClustersResponse)(nil), "docker.swarmkit.v1.ListClustersResponse") - proto.RegisterType((*JoinTokenRotation)(nil), "docker.swarmkit.v1.JoinTokenRotation") + proto.RegisterType((*KeyRotation)(nil), "docker.swarmkit.v1.KeyRotation") proto.RegisterType((*UpdateClusterRequest)(nil), "docker.swarmkit.v1.UpdateClusterRequest") proto.RegisterType((*UpdateClusterResponse)(nil), "docker.swarmkit.v1.UpdateClusterResponse") proto.RegisterType((*GetSecretRequest)(nil), "docker.swarmkit.v1.GetSecretRequest") @@ -1459,14 +1462,15 @@ func (m *ListClustersResponse) Copy() *ListClustersResponse { return o } -func (m *JoinTokenRotation) Copy() *JoinTokenRotation { +func (m *KeyRotation) Copy() *KeyRotation { if m == nil { return nil } - o := &JoinTokenRotation{ - RotateWorkerToken: m.RotateWorkerToken, - RotateManagerToken: m.RotateManagerToken, + o := &KeyRotation{ + WorkerJoinToken: m.WorkerJoinToken, + ManagerJoinToken: m.ManagerJoinToken, + ManagerUnlockKey: m.ManagerUnlockKey, } return o @@ -2199,14 +2203,15 @@ func (this *ListClustersResponse) GoString() string { s = append(s, "}") return strings.Join(s, "") } -func (this *JoinTokenRotation) GoString() string { +func (this *KeyRotation) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 6) - s = append(s, "&api.JoinTokenRotation{") - s = append(s, "RotateWorkerToken: "+fmt.Sprintf("%#v", this.RotateWorkerToken)+",\n") - s = append(s, "RotateManagerToken: "+fmt.Sprintf("%#v", this.RotateManagerToken)+",\n") + s := make([]string, 0, 7) + s = append(s, "&api.KeyRotation{") + s = append(s, "WorkerJoinToken: "+fmt.Sprintf("%#v", this.WorkerJoinToken)+",\n") + s = append(s, "ManagerJoinToken: "+fmt.Sprintf("%#v", this.ManagerJoinToken)+",\n") + s = append(s, "ManagerUnlockKey: "+fmt.Sprintf("%#v", this.ManagerUnlockKey)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4734,7 +4739,7 @@ func (m *ListClustersResponse) MarshalTo(data []byte) (int, error) { return i, nil } -func (m *JoinTokenRotation) Marshal() (data []byte, err error) { +func (m *KeyRotation) Marshal() (data []byte, err error) { size := m.Size() data = make([]byte, size) n, err := m.MarshalTo(data) @@ -4744,25 +4749,35 @@ func (m *JoinTokenRotation) Marshal() (data []byte, err error) { return data[:n], nil } -func (m *JoinTokenRotation) MarshalTo(data []byte) (int, error) { +func (m *KeyRotation) MarshalTo(data []byte) (int, error) { var i int _ = i var l int _ = l - if m.RotateWorkerToken { + if m.WorkerJoinToken { data[i] = 0x8 i++ - if m.RotateWorkerToken { + if m.WorkerJoinToken { data[i] = 1 } else { data[i] = 0 } i++ } - if m.RotateManagerToken { + if m.ManagerJoinToken { data[i] = 0x10 i++ - if m.RotateManagerToken { + if m.ManagerJoinToken { + data[i] = 1 + } else { + data[i] = 0 + } + i++ + } + if m.ManagerUnlockKey { + data[i] = 0x18 + i++ + if m.ManagerUnlockKey { data[i] = 1 } else { data[i] = 0 @@ -6618,13 +6633,16 @@ func (m *ListClustersResponse) Size() (n int) { return n } -func (m *JoinTokenRotation) Size() (n int) { +func (m *KeyRotation) Size() (n int) { var l int _ = l - if m.RotateWorkerToken { + if m.WorkerJoinToken { + n += 2 + } + if m.ManagerJoinToken { n += 2 } - if m.RotateManagerToken { + if m.ManagerUnlockKey { n += 2 } return n @@ -7294,13 +7312,14 @@ func (this *ListClustersResponse) String() string { }, "") return s } -func (this *JoinTokenRotation) String() string { +func (this *KeyRotation) String() string { if this == nil { return "nil" } - s := strings.Join([]string{`&JoinTokenRotation{`, - `RotateWorkerToken:` + fmt.Sprintf("%v", this.RotateWorkerToken) + `,`, - `RotateManagerToken:` + fmt.Sprintf("%v", this.RotateManagerToken) + `,`, + s := strings.Join([]string{`&KeyRotation{`, + `WorkerJoinToken:` + fmt.Sprintf("%v", this.WorkerJoinToken) + `,`, + `ManagerJoinToken:` + fmt.Sprintf("%v", this.ManagerJoinToken) + `,`, + `ManagerUnlockKey:` + fmt.Sprintf("%v", this.ManagerUnlockKey) + `,`, `}`, }, "") return s @@ -7313,7 +7332,7 @@ func (this *UpdateClusterRequest) String() string { `ClusterID:` + fmt.Sprintf("%v", this.ClusterID) + `,`, `ClusterVersion:` + strings.Replace(fmt.Sprintf("%v", this.ClusterVersion), "Version", "Version", 1) + `,`, `Spec:` + strings.Replace(fmt.Sprintf("%v", this.Spec), "ClusterSpec", "ClusterSpec", 1) + `,`, - `Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "JoinTokenRotation", "JoinTokenRotation", 1), `&`, ``, 1) + `,`, + `Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "KeyRotation", "KeyRotation", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -11855,7 +11874,7 @@ func (m *ListClustersResponse) Unmarshal(data []byte) error { } return nil } -func (m *JoinTokenRotation) Unmarshal(data []byte) error { +func (m *KeyRotation) Unmarshal(data []byte) error { l := len(data) iNdEx := 0 for iNdEx < l { @@ -11878,15 +11897,15 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: JoinTokenRotation: wiretype end group for non-group") + return fmt.Errorf("proto: KeyRotation: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: JoinTokenRotation: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: KeyRotation: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RotateWorkerToken", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field WorkerJoinToken", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -11903,10 +11922,30 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error { break } } - m.RotateWorkerToken = bool(v != 0) + m.WorkerJoinToken = bool(v != 0) case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RotateManagerToken", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ManagerJoinToken", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ManagerJoinToken = bool(v != 0) + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ManagerUnlockKey", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -11923,7 +11962,7 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error { break } } - m.RotateManagerToken = bool(v != 0) + m.ManagerUnlockKey = bool(v != 0) default: iNdEx = preIndex skippy, err := skipControl(data[iNdEx:]) @@ -13413,117 +13452,117 @@ var ( func init() { proto.RegisterFile("control.proto", fileDescriptorControl) } var fileDescriptorControl = []byte{ - // 1777 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcf, 0x6f, 0xdb, 0xc6, - 0x12, 0x8e, 0x24, 0xdb, 0xb2, 0x47, 0x96, 0x13, 0xaf, 0x95, 0x3c, 0x81, 0xc9, 0x93, 0x03, 0xe6, - 0xc5, 0x91, 0x81, 0x3c, 0x39, 0x4f, 0x79, 0x41, 0xd3, 0x14, 0xfd, 0x65, 0xbb, 0x71, 0x95, 0x1f, - 0x6e, 0x40, 0x27, 0x6d, 0x6f, 0x86, 0x2c, 0x6d, 0x5c, 0x56, 0xb2, 0xa8, 0x92, 0xb4, 0x93, 0xa0, - 0x97, 0x16, 0x68, 0x81, 0xfe, 0x09, 0xbd, 0xf6, 0xda, 0x02, 0x3d, 0xf7, 0xd6, 0x6b, 0xd0, 0x53, - 0x8f, 0x3d, 0x19, 0x8d, 0x80, 0x02, 0x3d, 0x15, 0xfd, 0x0b, 0x8a, 0x62, 0x77, 0x67, 0x49, 0x8a, - 0x5a, 0x92, 0x92, 0xe5, 0xc2, 0x39, 0x99, 0x5c, 0x7e, 0xb3, 0x33, 0xbb, 0xf3, 0xed, 0xa7, 0xd9, - 0x81, 0x21, 0xdf, 0xb0, 0x3a, 0xae, 0x6d, 0xb5, 0x2b, 0x5d, 0xdb, 0x72, 0x2d, 0x42, 0x9a, 0x56, - 0xa3, 0x45, 0xed, 0x8a, 0xf3, 0xa4, 0x6e, 0xef, 0xb5, 0x4c, 0xb7, 0x72, 0xf0, 0x3f, 0x2d, 0xe7, - 0x74, 0x69, 0xc3, 0x11, 0x00, 0x2d, 0x6f, 0xed, 0x7c, 0x4c, 0x1b, 0xae, 0x7c, 0xcd, 0xb9, 0xcf, - 0xba, 0x54, 0xbe, 0x14, 0x76, 0xad, 0x5d, 0x8b, 0x3f, 0xae, 0xb0, 0x27, 0x1c, 0x5d, 0xe8, 0xb6, - 0xf7, 0x77, 0xcd, 0xce, 0x8a, 0xf8, 0x23, 0x06, 0xf5, 0x1b, 0x30, 0xb7, 0x41, 0xdd, 0x4d, 0xab, - 0x49, 0x0d, 0xfa, 0xc9, 0x3e, 0x75, 0x5c, 0x72, 0x09, 0xb2, 0x1d, 0xab, 0x49, 0xb7, 0xcd, 0x66, - 0x31, 0x75, 0x31, 0x55, 0x9e, 0x59, 0x85, 0xde, 0xe1, 0xe2, 0x14, 0x43, 0xd4, 0xd6, 0x8d, 0x29, - 0xf6, 0xa9, 0xd6, 0xd4, 0xdf, 0x84, 0xd3, 0x9e, 0x99, 0xd3, 0xb5, 0x3a, 0x0e, 0x25, 0x57, 0x61, - 0x82, 0x7d, 0xe4, 0x46, 0xb9, 0x6a, 0xb1, 0x32, 0xb8, 0x80, 0x0a, 0xc7, 0x73, 0x94, 0x7e, 0x98, - 0x81, 0x33, 0xf7, 0x4c, 0x87, 0x4f, 0xe1, 0x48, 0xd7, 0xb7, 0x21, 0xfb, 0xd8, 0x6c, 0xbb, 0xd4, - 0x76, 0x70, 0x96, 0xab, 0xaa, 0x59, 0xc2, 0x66, 0x95, 0xdb, 0xc2, 0xc6, 0x90, 0xc6, 0xda, 0xe7, - 0x19, 0xc8, 0xe2, 0x20, 0x29, 0xc0, 0x64, 0xa7, 0xbe, 0x47, 0xd9, 0x8c, 0x99, 0xf2, 0x8c, 0x21, - 0x5e, 0xc8, 0x0a, 0xe4, 0xcc, 0xe6, 0x76, 0xd7, 0xa6, 0x8f, 0xcd, 0xa7, 0xd4, 0x29, 0xa6, 0xd9, - 0xb7, 0xd5, 0xb9, 0xde, 0xe1, 0x22, 0xd4, 0xd6, 0x1f, 0xe0, 0xa8, 0x01, 0x66, 0x53, 0x3e, 0x93, - 0x07, 0x30, 0xd5, 0xae, 0xef, 0xd0, 0xb6, 0x53, 0xcc, 0x5c, 0xcc, 0x94, 0x73, 0xd5, 0x9b, 0xa3, - 0x44, 0x56, 0xb9, 0xc7, 0x4d, 0xdf, 0xe9, 0xb8, 0xf6, 0x33, 0x03, 0xe7, 0x21, 0x35, 0xc8, 0xed, - 0xd1, 0xbd, 0x1d, 0x6a, 0x3b, 0x1f, 0x99, 0x5d, 0xa7, 0x38, 0x71, 0x31, 0x53, 0x9e, 0xab, 0x5e, - 0x89, 0xda, 0xb6, 0xad, 0x2e, 0x6d, 0x54, 0xee, 0x7b, 0x78, 0x23, 0x68, 0x4b, 0xaa, 0x30, 0x69, - 0x5b, 0x6d, 0xea, 0x14, 0x27, 0xf9, 0x24, 0x17, 0x22, 0xf7, 0xde, 0x6a, 0x53, 0x43, 0x40, 0xc9, - 0x25, 0xc8, 0xb3, 0xad, 0xf0, 0xf7, 0x60, 0x8a, 0xef, 0xcf, 0x2c, 0x1b, 0x94, 0xab, 0xd6, 0x5e, - 0x85, 0x5c, 0x20, 0x74, 0x72, 0x06, 0x32, 0x2d, 0xfa, 0x4c, 0xd0, 0xc2, 0x60, 0x8f, 0x6c, 0x77, - 0x0f, 0xea, 0xed, 0x7d, 0x5a, 0x4c, 0xf3, 0x31, 0xf1, 0x72, 0x2b, 0x7d, 0x33, 0xa5, 0xaf, 0xc1, - 0x7c, 0x60, 0x3b, 0x90, 0x23, 0x15, 0x98, 0x64, 0xd9, 0x17, 0xc9, 0x88, 0x23, 0x89, 0x80, 0xe9, - 0xdf, 0xa6, 0x60, 0xfe, 0x51, 0xb7, 0x59, 0x77, 0xe9, 0xa8, 0x0c, 0x25, 0x6f, 0xc0, 0x2c, 0x07, - 0x1d, 0x50, 0xdb, 0x31, 0xad, 0x0e, 0x0f, 0x30, 0x57, 0x3d, 0xaf, 0xf2, 0xf8, 0xbe, 0x80, 0x18, - 0x39, 0x66, 0x80, 0x2f, 0xe4, 0x1a, 0x4c, 0xb0, 0xe3, 0x56, 0xcc, 0x70, 0xbb, 0x0b, 0x71, 0x79, - 0x31, 0x38, 0x52, 0x5f, 0x05, 0x12, 0x8c, 0xf5, 0x48, 0xc7, 0x62, 0x13, 0xe6, 0x0d, 0xba, 0x67, - 0x1d, 0x8c, 0xbe, 0xde, 0x02, 0x4c, 0x3e, 0xb6, 0xec, 0x86, 0xc8, 0xc4, 0xb4, 0x21, 0x5e, 0xf4, - 0x02, 0x90, 0xe0, 0x7c, 0x22, 0x26, 0x3c, 0xf4, 0x0f, 0xeb, 0x4e, 0x2b, 0xe0, 0xc2, 0xad, 0x3b, - 0xad, 0x90, 0x0b, 0x86, 0x60, 0x2e, 0xd8, 0x27, 0xef, 0xd0, 0x0b, 0x33, 0x7f, 0x75, 0xec, 0x63, - 0xdc, 0xea, 0x38, 0x9e, 0xa3, 0xf4, 0x9b, 0x72, 0x75, 0x23, 0xbb, 0xf6, 0xd6, 0x11, 0xf4, 0xae, - 0xff, 0x85, 0x22, 0xc2, 0x06, 0x8f, 0x20, 0x22, 0x41, 0xb3, 0x41, 0x11, 0xf9, 0xe6, 0x04, 0x45, - 0x44, 0x15, 0x99, 0x52, 0x44, 0x56, 0x20, 0xe7, 0x50, 0xfb, 0xc0, 0x6c, 0x30, 0x76, 0x08, 0x11, - 0xc1, 0x10, 0xb6, 0xc4, 0x70, 0x6d, 0xdd, 0x31, 0x00, 0x21, 0xb5, 0xa6, 0x43, 0x96, 0x60, 0x1a, - 0xb9, 0x24, 0xd4, 0x62, 0x66, 0x35, 0xd7, 0x3b, 0x5c, 0xcc, 0x0a, 0x32, 0x39, 0x46, 0x56, 0xb0, - 0xc9, 0x21, 0xeb, 0x30, 0xd7, 0xa4, 0x8e, 0x69, 0xd3, 0xe6, 0xb6, 0xe3, 0xd6, 0x5d, 0xd4, 0x87, - 0xb9, 0xea, 0xbf, 0xa3, 0x52, 0xbc, 0xc5, 0x50, 0x46, 0x1e, 0x8d, 0xf8, 0x9b, 0x42, 0x64, 0xb2, - 0xff, 0x88, 0xc8, 0xe0, 0x76, 0xf9, 0x22, 0xc3, 0x58, 0x13, 0x2b, 0x32, 0x9c, 0x46, 0x02, 0xa6, - 0xdf, 0x85, 0xc2, 0x9a, 0x4d, 0xeb, 0x2e, 0xc5, 0x2d, 0x93, 0x44, 0xba, 0x8e, 0x0a, 0x20, 0x58, - 0xb4, 0xa8, 0x9a, 0x06, 0x2d, 0x02, 0x22, 0xb0, 0x09, 0x67, 0x43, 0x93, 0x61, 0x54, 0x37, 0x20, - 0x8b, 0x69, 0xc0, 0x09, 0xcf, 0xc7, 0x4c, 0x68, 0x48, 0xac, 0xfe, 0x36, 0xcc, 0x6f, 0x50, 0x37, - 0x14, 0xd9, 0x55, 0x00, 0x3f, 0xeb, 0x78, 0x6a, 0xf2, 0xbd, 0xc3, 0xc5, 0x19, 0x2f, 0xe9, 0xc6, - 0x8c, 0x97, 0x73, 0xfd, 0x2e, 0x90, 0xe0, 0x14, 0xe3, 0xc5, 0xf3, 0x63, 0x0a, 0x0a, 0x42, 0xe5, - 0xc6, 0x89, 0x89, 0xac, 0xc3, 0x69, 0x89, 0x1e, 0x41, 0xa0, 0xe7, 0xd0, 0x46, 0x6a, 0xf4, 0xf5, - 0x3e, 0x8d, 0x1e, 0x3e, 0x43, 0xa1, 0x05, 0x8c, 0xb7, 0x23, 0xeb, 0x50, 0x10, 0xd2, 0x34, 0x56, - 0x92, 0xfe, 0x05, 0x67, 0x43, 0xb3, 0xa0, 0xc6, 0xfd, 0x9e, 0x86, 0x05, 0xc6, 0x71, 0x1c, 0xf7, - 0x64, 0xae, 0x16, 0x96, 0xb9, 0x95, 0x28, 0x31, 0x09, 0x59, 0x0e, 0x2a, 0xdd, 0x97, 0xe9, 0x63, - 0x57, 0xba, 0xad, 0x90, 0xd2, 0xbd, 0x36, 0x62, 0x70, 0x4a, 0xb1, 0x1b, 0x50, 0x93, 0x89, 0xe3, - 0x55, 0x93, 0xf7, 0xa0, 0xd0, 0x1f, 0x12, 0x12, 0xe3, 0x15, 0x98, 0xc6, 0x44, 0x49, 0x4d, 0x89, - 0x65, 0x86, 0x07, 0xf6, 0x95, 0x65, 0x93, 0xba, 0x4f, 0x2c, 0xbb, 0x35, 0x82, 0xb2, 0xa0, 0x85, - 0x4a, 0x59, 0xbc, 0xc9, 0x7c, 0xde, 0x76, 0xc4, 0x50, 0x1c, 0x6f, 0xa5, 0x95, 0xc4, 0xea, 0x8f, - 0xb8, 0xb2, 0x84, 0x22, 0x23, 0x30, 0xc1, 0x76, 0x13, 0xf7, 0x8b, 0x3f, 0x33, 0x22, 0xa3, 0x0d, - 0x23, 0x72, 0xda, 0x27, 0x32, 0xda, 0x32, 0x22, 0x23, 0xc0, 0x53, 0x9b, 0x63, 0x8a, 0xf1, 0x43, - 0x79, 0xb6, 0x8e, 0x3d, 0x4c, 0xef, 0xbc, 0x85, 0x22, 0xf5, 0xce, 0x1b, 0x8e, 0x1f, 0xe1, 0xbc, - 0x85, 0x2c, 0x5f, 0xae, 0xf3, 0x16, 0x11, 0xdc, 0x49, 0x9e, 0x37, 0x3f, 0x24, 0xff, 0xbc, 0x61, - 0xa2, 0x62, 0xcf, 0x9b, 0xcc, 0x9c, 0x07, 0xc6, 0x1f, 0xcb, 0xb5, 0xf6, 0xbe, 0xe3, 0x52, 0x3b, - 0xa0, 0xc3, 0x0d, 0x31, 0x12, 0xd2, 0x61, 0xc4, 0x31, 0x5e, 0x20, 0xc0, 0xa3, 0xaf, 0x37, 0x85, - 0x4f, 0x5f, 0x84, 0xc4, 0xd1, 0x57, 0x5a, 0x49, 0xac, 0xc7, 0x25, 0xfc, 0x70, 0x04, 0x2e, 0x85, - 0x2c, 0x5f, 0x2e, 0x2e, 0x45, 0x04, 0x77, 0x92, 0x5c, 0xf2, 0x43, 0xf2, 0xb9, 0x84, 0xd9, 0x88, - 0xe5, 0x92, 0x4c, 0x9d, 0x07, 0xd6, 0xf7, 0x61, 0xfe, 0x8e, 0x65, 0x76, 0x1e, 0x5a, 0x2d, 0xda, - 0x31, 0x2c, 0xb7, 0xee, 0xb2, 0x82, 0xa3, 0x02, 0x0b, 0x36, 0x7b, 0xa6, 0xdb, 0x8c, 0x70, 0xd4, - 0xde, 0x76, 0xd9, 0x67, 0x1e, 0xe1, 0xb4, 0x31, 0x2f, 0x3e, 0x7d, 0xc0, 0xbf, 0x70, 0x3b, 0x72, - 0x0d, 0x0a, 0x88, 0xdf, 0xab, 0x77, 0xea, 0xbb, 0x9e, 0x81, 0xb8, 0xa3, 0x11, 0xf1, 0xed, 0xbe, - 0xf8, 0xc4, 0x2d, 0xf4, 0xaf, 0xd2, 0xb2, 0xbe, 0x1a, 0x87, 0xc6, 0xac, 0xbe, 0x92, 0xe8, 0x51, - 0xea, 0x2b, 0xb4, 0x19, 0xa1, 0xbe, 0x42, 0xef, 0xfe, 0xef, 0x14, 0xd9, 0x80, 0x69, 0x1b, 0xf7, - 0xab, 0x38, 0xc1, 0x0d, 0x2f, 0xab, 0x0c, 0x07, 0x36, 0x77, 0x75, 0xe2, 0xf9, 0xe1, 0xe2, 0x29, - 0xc3, 0x33, 0xf6, 0x0b, 0xb5, 0x63, 0x3a, 0x8d, 0xaf, 0xc3, 0x19, 0x5e, 0x07, 0x37, 0x6c, 0xea, - 0xca, 0x5d, 0x5d, 0x86, 0x19, 0x87, 0x0f, 0xf8, 0x9b, 0x3a, 0xdb, 0x3b, 0x5c, 0x9c, 0x16, 0xa8, - 0xda, 0x3a, 0xfb, 0x31, 0xe7, 0x4f, 0x4d, 0x7d, 0x03, 0x2b, 0x71, 0x61, 0x8e, 0xa1, 0x54, 0x61, - 0x4a, 0x00, 0x30, 0x12, 0x4d, 0x5d, 0x18, 0x70, 0x1b, 0x44, 0xea, 0x3f, 0xa4, 0x60, 0x41, 0x56, - 0xa0, 0x47, 0x8b, 0x85, 0xac, 0xc2, 0x1c, 0x42, 0x47, 0xc8, 0x6e, 0x5e, 0x98, 0xc8, 0xe4, 0x56, - 0xfb, 0x92, 0x5b, 0x8a, 0x0e, 0x3c, 0x50, 0x83, 0xdc, 0xf1, 0x8b, 0xff, 0xb1, 0xb7, 0xe1, 0xb7, - 0x34, 0x10, 0x51, 0x6e, 0xb1, 0x57, 0x4f, 0x1b, 0xdf, 0x0d, 0x6b, 0x63, 0x25, 0xba, 0x74, 0x0c, - 0x1a, 0x0e, 0x4a, 0xe3, 0x17, 0xc7, 0x2f, 0x8d, 0x46, 0x48, 0x1a, 0x6f, 0x8d, 0x16, 0xdb, 0x89, - 0x28, 0xe3, 0x5d, 0x79, 0x7f, 0xc0, 0x88, 0x30, 0x65, 0xff, 0x67, 0xb7, 0x1d, 0x3e, 0x84, 0xba, - 0x18, 0x97, 0x33, 0x09, 0xd5, 0x6b, 0xb0, 0x20, 0xaf, 0xb7, 0x41, 0xea, 0x56, 0xfb, 0x0a, 0xda, - 0xa1, 0xb9, 0xd4, 0x3f, 0xd5, 0x18, 0x5c, 0x7a, 0x0b, 0x16, 0xe4, 0xed, 0xe9, 0x88, 0xa7, 0xfb, - 0x9c, 0x7f, 0x8b, 0x0b, 0x46, 0x53, 0xfd, 0xee, 0x1c, 0x64, 0xd7, 0x44, 0x67, 0x9e, 0x98, 0x90, - 0xc5, 0xa6, 0x37, 0xd1, 0x55, 0x41, 0xf5, 0x37, 0xd2, 0xb5, 0x4b, 0xb1, 0x18, 0x2c, 0x37, 0xcf, - 0xfe, 0xf4, 0xfd, 0x1f, 0x5f, 0xa7, 0x4f, 0x43, 0x9e, 0x83, 0xfe, 0x8b, 0x3f, 0x13, 0xc4, 0x82, - 0x19, 0xaf, 0x7b, 0x4a, 0xfe, 0x33, 0x4c, 0xaf, 0x59, 0xbb, 0x9c, 0x80, 0x8a, 0x77, 0x68, 0x03, - 0xf8, 0xcd, 0x4b, 0xa2, 0x9c, 0x6b, 0xa0, 0x11, 0xab, 0x2d, 0x25, 0xc1, 0x12, 0x7d, 0xfa, 0xcd, - 0x49, 0xb5, 0xcf, 0x81, 0x66, 0xa8, 0xda, 0xa7, 0xa2, 0xc7, 0x19, 0xe1, 0x53, 0xe4, 0xf0, 0x61, - 0xdd, 0x69, 0x45, 0xe6, 0x30, 0xd0, 0x9c, 0x8c, 0xcc, 0x61, 0x5f, 0x1b, 0x32, 0x3e, 0x87, 0xbc, - 0x39, 0x15, 0x9d, 0xc3, 0x60, 0xab, 0x2f, 0x3a, 0x87, 0x7d, 0x1d, 0xae, 0xc4, 0xfd, 0xe4, 0xcb, - 0x8b, 0xd9, 0xcf, 0xe0, 0x0a, 0x97, 0x92, 0x60, 0x89, 0x3e, 0xfd, 0xe6, 0x92, 0xda, 0xe7, 0x40, - 0xff, 0x4a, 0xed, 0x73, 0xb0, 0x47, 0x15, 0xe5, 0xf3, 0x29, 0xcc, 0x06, 0xef, 0xe9, 0xe4, 0xca, - 0x90, 0xcd, 0x05, 0xad, 0x9c, 0x0c, 0x8c, 0xf7, 0xfc, 0x29, 0xe4, 0xfb, 0xba, 0x7b, 0x44, 0x39, - 0xa3, 0xaa, 0x9b, 0xa8, 0x2d, 0x0f, 0x81, 0x4c, 0x74, 0xde, 0xd7, 0xb8, 0x52, 0x3b, 0x57, 0x35, - 0xe7, 0xd4, 0xce, 0x95, 0x5d, 0xb0, 0x18, 0xe7, 0x7d, 0xfd, 0x29, 0xb5, 0x73, 0x55, 0x23, 0x4c, - 0xed, 0x5c, 0xdd, 0xec, 0x8a, 0x25, 0x19, 0xde, 0xf7, 0x22, 0x49, 0xd6, 0xdf, 0x23, 0x88, 0x24, - 0x59, 0xf8, 0xc2, 0x1f, 0x4f, 0x32, 0x79, 0x39, 0x8d, 0x26, 0x59, 0xe8, 0x46, 0x1d, 0x4d, 0xb2, - 0xf0, 0x3d, 0x37, 0x91, 0x64, 0x72, 0xc1, 0x31, 0x24, 0x0b, 0xad, 0x79, 0x79, 0x08, 0xe4, 0x90, - 0x79, 0x8e, 0x75, 0xae, 0x6a, 0xca, 0xc4, 0xe5, 0x79, 0x48, 0xe7, 0x22, 0xcf, 0x58, 0xb8, 0x47, - 0xe6, 0xb9, 0xff, 0x62, 0x14, 0x99, 0xe7, 0xd0, 0xad, 0x21, 0x21, 0xcf, 0xf2, 0xe2, 0x18, 0x9d, - 0xe7, 0xd0, 0x6d, 0x37, 0x3a, 0xcf, 0xe1, 0x3b, 0x68, 0xe2, 0x79, 0x96, 0x0b, 0x8e, 0x39, 0xcf, - 0xa1, 0x35, 0x2f, 0x0f, 0x81, 0x4c, 0xfc, 0x71, 0xf2, 0x6e, 0x33, 0xea, 0x1f, 0xa7, 0xf0, 0x5d, - 0x49, 0xbb, 0x9c, 0x80, 0x4a, 0xdc, 0xe7, 0xe0, 0xd5, 0x41, 0xbd, 0xcf, 0x8a, 0x6b, 0x91, 0x56, - 0x4e, 0x06, 0xc6, 0x7b, 0xde, 0x87, 0x5c, 0xa0, 0x00, 0x26, 0x4b, 0xc3, 0xd5, 0xec, 0xda, 0x95, - 0x44, 0x5c, 0xe2, 0x82, 0x83, 0xf5, 0xad, 0x7a, 0xc1, 0x8a, 0x62, 0x5a, 0x2b, 0x27, 0x03, 0x13, - 0x3d, 0x07, 0x6b, 0x59, 0xb5, 0x67, 0x45, 0xbd, 0xac, 0x95, 0x93, 0x81, 0xb1, 0x9e, 0x57, 0x2f, - 0x3c, 0x7f, 0x51, 0x3a, 0xf5, 0xcb, 0x8b, 0xd2, 0xa9, 0x3f, 0x5f, 0x94, 0x52, 0x9f, 0xf5, 0x4a, - 0xa9, 0xe7, 0xbd, 0x52, 0xea, 0xe7, 0x5e, 0x29, 0xf5, 0x6b, 0xaf, 0x94, 0xda, 0x99, 0xe2, 0xff, - 0x72, 0x72, 0xfd, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2a, 0x7c, 0x4c, 0x3e, 0xeb, 0x22, 0x00, - 0x00, + // 1781 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x6f, 0x1b, 0x45, + 0x14, 0xaf, 0xed, 0x24, 0x4e, 0x9e, 0xe3, 0x7c, 0x4c, 0xdc, 0x62, 0x6d, 0x8b, 0x53, 0x6d, 0x69, + 0xea, 0xa0, 0xe0, 0x80, 0x4b, 0x45, 0x29, 0xe2, 0xa3, 0x8e, 0x69, 0x71, 0x53, 0x42, 0xb5, 0x69, + 0x11, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0x6b, 0xc7, 0x6b, 0x76, 0x37, 0x69, 0x23, 0x2e, 0x80, 0xe0, + 0x4f, 0x40, 0xe2, 0xca, 0x15, 0x24, 0xce, 0xdc, 0xb8, 0x56, 0x9c, 0x38, 0x72, 0xb2, 0xa8, 0x25, + 0x24, 0x4e, 0x88, 0xbf, 0x00, 0xa1, 0xf9, 0xda, 0x2f, 0xcf, 0xee, 0xda, 0x71, 0x50, 0x7a, 0x8a, + 0x77, 0xf6, 0xf7, 0xe6, 0xbd, 0x99, 0xf7, 0x9b, 0xdf, 0xbe, 0x79, 0x0a, 0x64, 0x1b, 0x46, 0xc7, + 0x36, 0x8d, 0x76, 0xa9, 0x6b, 0x1a, 0xb6, 0x81, 0x50, 0xd3, 0x68, 0xb4, 0xb0, 0x59, 0xb2, 0x1e, + 0xd7, 0xcd, 0xfd, 0x96, 0x6e, 0x97, 0x0e, 0x5f, 0x53, 0x32, 0x56, 0x17, 0x37, 0x2c, 0x06, 0x50, + 0xb2, 0xc6, 0xee, 0x23, 0xdc, 0xb0, 0xc5, 0x63, 0xc6, 0x3e, 0xea, 0x62, 0xf1, 0x90, 0xdb, 0x33, + 0xf6, 0x0c, 0xfa, 0x73, 0x9d, 0xfc, 0xe2, 0xa3, 0x4b, 0xdd, 0xf6, 0xc1, 0x9e, 0xde, 0x59, 0x67, + 0x7f, 0xd8, 0xa0, 0x7a, 0x0d, 0xe6, 0x6e, 0x63, 0x7b, 0xcb, 0x68, 0x62, 0x0d, 0x7f, 0x76, 0x80, + 0x2d, 0x1b, 0x5d, 0x82, 0x74, 0xc7, 0x68, 0xe2, 0x1d, 0xbd, 0x99, 0x4f, 0x5c, 0x4c, 0x14, 0x67, + 0x2a, 0xd0, 0xef, 0x2d, 0x4f, 0x11, 0x44, 0xad, 0xaa, 0x4d, 0x91, 0x57, 0xb5, 0xa6, 0xfa, 0x2e, + 0xcc, 0x3b, 0x66, 0x56, 0xd7, 0xe8, 0x58, 0x18, 0xad, 0xc1, 0x04, 0x79, 0x49, 0x8d, 0x32, 0xe5, + 0x7c, 0x69, 0x70, 0x01, 0x25, 0x8a, 0xa7, 0x28, 0xb5, 0x97, 0x82, 0x85, 0xbb, 0xba, 0x45, 0xa7, + 0xb0, 0x84, 0xeb, 0x5b, 0x90, 0x7e, 0xa8, 0xb7, 0x6d, 0x6c, 0x5a, 0x7c, 0x96, 0x35, 0xd9, 0x2c, + 0x41, 0xb3, 0xd2, 0x2d, 0x66, 0xa3, 0x09, 0x63, 0xe5, 0xcb, 0x14, 0xa4, 0xf9, 0x20, 0xca, 0xc1, + 0x64, 0xa7, 0xbe, 0x8f, 0xc9, 0x8c, 0xa9, 0xe2, 0x8c, 0xc6, 0x1e, 0xd0, 0x3a, 0x64, 0xf4, 0xe6, + 0x4e, 0xd7, 0xc4, 0x0f, 0xf5, 0x27, 0xd8, 0xca, 0x27, 0xc9, 0xbb, 0xca, 0x5c, 0xbf, 0xb7, 0x0c, + 0xb5, 0xea, 0x3d, 0x3e, 0xaa, 0x81, 0xde, 0x14, 0xbf, 0xd1, 0x3d, 0x98, 0x6a, 0xd7, 0x77, 0x71, + 0xdb, 0xca, 0xa7, 0x2e, 0xa6, 0x8a, 0x99, 0xf2, 0xf5, 0x51, 0x22, 0x2b, 0xdd, 0xa5, 0xa6, 0xef, + 0x77, 0x6c, 0xf3, 0x48, 0xe3, 0xf3, 0xa0, 0x1a, 0x64, 0xf6, 0xf1, 0xfe, 0x2e, 0x36, 0xad, 0x4f, + 0xf5, 0xae, 0x95, 0x9f, 0xb8, 0x98, 0x2a, 0xce, 0x95, 0xaf, 0x84, 0x6d, 0xdb, 0x76, 0x17, 0x37, + 0x4a, 0x1f, 0x3a, 0x78, 0xcd, 0x6b, 0x8b, 0xca, 0x30, 0x69, 0x1a, 0x6d, 0x6c, 0xe5, 0x27, 0xe9, + 0x24, 0x17, 0x42, 0xf7, 0xde, 0x68, 0x63, 0x8d, 0x41, 0xd1, 0x25, 0xc8, 0x92, 0xad, 0x70, 0xf7, + 0x60, 0x8a, 0xee, 0xcf, 0x2c, 0x19, 0x14, 0xab, 0x56, 0xde, 0x84, 0x8c, 0x27, 0x74, 0xb4, 0x00, + 0xa9, 0x16, 0x3e, 0x62, 0xb4, 0xd0, 0xc8, 0x4f, 0xb2, 0xbb, 0x87, 0xf5, 0xf6, 0x01, 0xce, 0x27, + 0xe9, 0x18, 0x7b, 0xb8, 0x91, 0xbc, 0x9e, 0x50, 0x37, 0x60, 0xd1, 0xb3, 0x1d, 0x9c, 0x23, 0x25, + 0x98, 0x24, 0xd9, 0x67, 0xc9, 0x88, 0x22, 0x09, 0x83, 0xa9, 0x3f, 0x24, 0x60, 0xf1, 0x41, 0xb7, + 0x59, 0xb7, 0xf1, 0xa8, 0x0c, 0x45, 0xef, 0xc0, 0x2c, 0x05, 0x1d, 0x62, 0xd3, 0xd2, 0x8d, 0x0e, + 0x0d, 0x30, 0x53, 0x3e, 0x2f, 0xf3, 0xf8, 0x31, 0x83, 0x68, 0x19, 0x62, 0xc0, 0x1f, 0xd0, 0xab, + 0x30, 0x41, 0x8e, 0x5b, 0x3e, 0x45, 0xed, 0x2e, 0x44, 0xe5, 0x45, 0xa3, 0x48, 0xb5, 0x02, 0xc8, + 0x1b, 0xeb, 0xb1, 0x8e, 0xc5, 0x16, 0x2c, 0x6a, 0x78, 0xdf, 0x38, 0x1c, 0x7d, 0xbd, 0x39, 0x98, + 0x7c, 0x68, 0x98, 0x0d, 0x96, 0x89, 0x69, 0x8d, 0x3d, 0xa8, 0x39, 0x40, 0xde, 0xf9, 0x58, 0x4c, + 0xfc, 0xd0, 0xdf, 0xaf, 0x5b, 0x2d, 0x8f, 0x0b, 0xbb, 0x6e, 0xb5, 0x02, 0x2e, 0x08, 0x82, 0xb8, + 0x20, 0xaf, 0x9c, 0x43, 0xcf, 0xcc, 0xdc, 0xd5, 0x91, 0x97, 0x51, 0xab, 0xa3, 0x78, 0x8a, 0x52, + 0xaf, 0x8b, 0xd5, 0x8d, 0xec, 0xda, 0x59, 0x87, 0xd7, 0xbb, 0xfa, 0x2f, 0x17, 0x11, 0x32, 0x78, + 0x0c, 0x11, 0xf1, 0x9a, 0x0d, 0x8a, 0xc8, 0xf7, 0xa7, 0x28, 0x22, 0xb2, 0xc8, 0xa4, 0x22, 0xb2, + 0x0e, 0x19, 0x0b, 0x9b, 0x87, 0x7a, 0x83, 0xb0, 0x83, 0x89, 0x08, 0x0f, 0x61, 0x9b, 0x0d, 0xd7, + 0xaa, 0x96, 0x06, 0x1c, 0x52, 0x6b, 0x5a, 0x68, 0x05, 0xa6, 0x39, 0x97, 0x98, 0x5a, 0xcc, 0x54, + 0x32, 0xfd, 0xde, 0x72, 0x9a, 0x91, 0xc9, 0xd2, 0xd2, 0x8c, 0x4d, 0x16, 0xaa, 0xc2, 0x5c, 0x13, + 0x5b, 0xba, 0x89, 0x9b, 0x3b, 0x96, 0x5d, 0xb7, 0xb9, 0x3e, 0xcc, 0x95, 0x5f, 0x0c, 0x4b, 0xf1, + 0x36, 0x41, 0x69, 0x59, 0x6e, 0x44, 0x9f, 0x24, 0x22, 0x93, 0xfe, 0x5f, 0x44, 0x86, 0x6f, 0x97, + 0x2b, 0x32, 0x84, 0x35, 0x91, 0x22, 0x43, 0x69, 0xc4, 0x60, 0xea, 0x26, 0xe4, 0x36, 0x4c, 0x5c, + 0xb7, 0x31, 0xdf, 0x32, 0x41, 0xa4, 0xab, 0x5c, 0x01, 0x18, 0x8b, 0x96, 0x65, 0xd3, 0x70, 0x0b, + 0x8f, 0x08, 0x6c, 0xc1, 0xd9, 0xc0, 0x64, 0x3c, 0xaa, 0x6b, 0x90, 0xe6, 0x69, 0xe0, 0x13, 0x9e, + 0x8f, 0x98, 0x50, 0x13, 0x58, 0xf5, 0x26, 0x2c, 0xde, 0xc6, 0x76, 0x20, 0xb2, 0x35, 0x00, 0x37, + 0xeb, 0xfc, 0xd4, 0x64, 0xfb, 0xbd, 0xe5, 0x19, 0x27, 0xe9, 0xda, 0x8c, 0x93, 0x73, 0x75, 0x13, + 0x90, 0x77, 0x8a, 0xf1, 0xe2, 0xf9, 0x25, 0x01, 0x39, 0xa6, 0x72, 0xe3, 0xc4, 0x84, 0xaa, 0x30, + 0x2f, 0xd0, 0x23, 0x08, 0xf4, 0x1c, 0xb7, 0x11, 0x1a, 0x7d, 0xd5, 0xa7, 0xd1, 0xc3, 0x67, 0x28, + 0xb0, 0x80, 0xf1, 0x76, 0xa4, 0x0a, 0x39, 0x26, 0x4d, 0x63, 0x25, 0xe9, 0x05, 0x38, 0x1b, 0x98, + 0x85, 0x6b, 0xdc, 0x5f, 0x49, 0x58, 0x22, 0x1c, 0xe7, 0xe3, 0x8e, 0xcc, 0xd5, 0x82, 0x32, 0xb7, + 0x1e, 0x26, 0x26, 0x01, 0xcb, 0x41, 0xa5, 0xfb, 0x26, 0x79, 0xe2, 0x4a, 0xb7, 0x1d, 0x50, 0xba, + 0xb7, 0x46, 0x0c, 0x4e, 0x2a, 0x76, 0x03, 0x6a, 0x32, 0x71, 0xb2, 0x6a, 0xf2, 0x11, 0xe4, 0xfc, + 0x21, 0x71, 0x62, 0xbc, 0x01, 0xd3, 0x3c, 0x51, 0x42, 0x53, 0x22, 0x99, 0xe1, 0x80, 0x5d, 0x65, + 0xd9, 0xc2, 0xf6, 0x63, 0xc3, 0x6c, 0x8d, 0xa0, 0x2c, 0xdc, 0x42, 0xa6, 0x2c, 0xce, 0x64, 0x2e, + 0x6f, 0x3b, 0x6c, 0x28, 0x8a, 0xb7, 0xc2, 0x4a, 0x60, 0xd5, 0x07, 0x54, 0x59, 0x02, 0x91, 0x21, + 0x98, 0x20, 0xbb, 0xc9, 0xf7, 0x8b, 0xfe, 0x26, 0x44, 0xe6, 0x36, 0x84, 0xc8, 0x49, 0x97, 0xc8, + 0xdc, 0x96, 0x10, 0x99, 0x03, 0x1c, 0xb5, 0x39, 0xa1, 0x18, 0x3f, 0x11, 0x67, 0xeb, 0xc4, 0xc3, + 0x74, 0xce, 0x5b, 0x20, 0x52, 0xe7, 0xbc, 0xf1, 0xf1, 0x63, 0x9c, 0xb7, 0x80, 0xe5, 0xf3, 0x75, + 0xde, 0x42, 0x82, 0x3b, 0xcd, 0xf3, 0xe6, 0x86, 0xe4, 0x9e, 0x37, 0x9e, 0xa8, 0xc8, 0xf3, 0x26, + 0x32, 0xe7, 0x80, 0xf9, 0xc7, 0x72, 0xa3, 0x7d, 0x60, 0xd9, 0xd8, 0xf4, 0xe8, 0x70, 0x83, 0x8d, + 0x04, 0x74, 0x98, 0xe3, 0x08, 0x2f, 0x38, 0xc0, 0xa1, 0xaf, 0x33, 0x85, 0x4b, 0x5f, 0x0e, 0x89, + 0xa2, 0xaf, 0xb0, 0x12, 0x58, 0x87, 0x4b, 0xfc, 0xc5, 0x31, 0xb8, 0x14, 0xb0, 0x7c, 0xbe, 0xb8, + 0x14, 0x12, 0xdc, 0x69, 0x72, 0xc9, 0x0d, 0xc9, 0xe5, 0x12, 0xcf, 0x46, 0x24, 0x97, 0x44, 0xea, + 0x1c, 0xb0, 0xfa, 0x6d, 0x02, 0x32, 0x9b, 0xf8, 0x48, 0x33, 0xec, 0xba, 0x4d, 0x6a, 0x8d, 0x97, + 0x61, 0x91, 0x90, 0x0c, 0x9b, 0x3b, 0x8f, 0x0c, 0xbd, 0xb3, 0x63, 0x1b, 0x2d, 0xdc, 0xa1, 0xa1, + 0x4d, 0x6b, 0xf3, 0xec, 0xc5, 0x1d, 0x43, 0xef, 0xdc, 0x27, 0xc3, 0x68, 0x0d, 0xd0, 0x7e, 0xbd, + 0x53, 0xdf, 0xf3, 0x83, 0xd9, 0xc5, 0x6c, 0x81, 0xbf, 0x91, 0xa2, 0x0f, 0x3a, 0x6d, 0xa3, 0xd1, + 0xda, 0x21, 0xab, 0x4e, 0xf9, 0xd0, 0x0f, 0xe8, 0x8b, 0x4d, 0x7c, 0xa4, 0x7e, 0x95, 0x14, 0x05, + 0xd8, 0x38, 0x3c, 0x27, 0x05, 0x98, 0x40, 0x8f, 0x52, 0x80, 0x71, 0x9b, 0x11, 0x0a, 0x30, 0xee, + 0xdd, 0xfd, 0x90, 0xa1, 0x9b, 0x30, 0x6d, 0xf2, 0x5d, 0xcd, 0x4f, 0x84, 0x1b, 0x7a, 0x36, 0xbf, + 0x32, 0xf1, 0xb4, 0xb7, 0x7c, 0x46, 0x73, 0xcc, 0xdc, 0x1a, 0xee, 0x84, 0x0e, 0xea, 0xdb, 0xb0, + 0x40, 0x4b, 0xe4, 0x86, 0x89, 0x6d, 0xb1, 0x9f, 0xab, 0x30, 0x63, 0xd1, 0x01, 0x77, 0x3b, 0x67, + 0xfb, 0xbd, 0xe5, 0x69, 0x86, 0xaa, 0x55, 0xc9, 0x77, 0x9e, 0xfe, 0x6a, 0xaa, 0xb7, 0x79, 0x91, + 0xce, 0xcc, 0x79, 0x28, 0x65, 0x98, 0x62, 0x00, 0x1e, 0x89, 0x22, 0xaf, 0x19, 0xa8, 0x0d, 0x47, + 0xaa, 0x3f, 0x27, 0x60, 0x49, 0x14, 0xa7, 0xc7, 0x8b, 0x05, 0x55, 0x60, 0x8e, 0x43, 0x47, 0xc8, + 0x6b, 0x96, 0x99, 0x88, 0xb4, 0x96, 0x7d, 0x69, 0x2d, 0x84, 0x07, 0xee, 0x29, 0x4f, 0xee, 0xb8, + 0xf7, 0x82, 0xb1, 0xb7, 0xe1, 0xcf, 0x24, 0x20, 0x56, 0x89, 0x91, 0x47, 0x47, 0x36, 0x3f, 0x08, + 0xca, 0x66, 0x29, 0xbc, 0xaa, 0xf4, 0x1a, 0x0e, 0xaa, 0xe6, 0xd7, 0x27, 0xaf, 0x9a, 0x5a, 0x40, + 0x35, 0x6f, 0x8c, 0x16, 0xdb, 0xa9, 0x88, 0xe6, 0xa6, 0xb8, 0x5a, 0xf0, 0x88, 0x78, 0xca, 0x5e, + 0x27, 0x17, 0x21, 0x3a, 0xc4, 0x25, 0x33, 0x2a, 0x67, 0x02, 0xaa, 0xd6, 0x60, 0x49, 0xdc, 0x7c, + 0xbd, 0xd4, 0x2d, 0xfb, 0x6a, 0xdd, 0xa1, 0xb9, 0xe4, 0x9f, 0x6a, 0x0c, 0x2e, 0xbd, 0x07, 0x4b, + 0xe2, 0x62, 0x75, 0xcc, 0xd3, 0x7d, 0xce, 0xbd, 0xe0, 0x79, 0xa3, 0x29, 0xff, 0x78, 0x0e, 0xd2, + 0x1b, 0xac, 0x69, 0x8f, 0x74, 0x48, 0xf3, 0x7e, 0x38, 0x52, 0x65, 0x41, 0xf9, 0x7b, 0xec, 0xca, + 0xa5, 0x48, 0x0c, 0xaf, 0x44, 0xcf, 0xfe, 0xfa, 0xd3, 0xdf, 0xdf, 0x25, 0xe7, 0x21, 0x4b, 0x41, + 0xaf, 0xf0, 0x2f, 0x01, 0x32, 0x60, 0xc6, 0x69, 0xac, 0xa2, 0x97, 0x86, 0x69, 0x43, 0x2b, 0x97, + 0x63, 0x50, 0xd1, 0x0e, 0x4d, 0x00, 0xb7, 0xaf, 0x89, 0xa4, 0x73, 0x0d, 0xf4, 0x68, 0x95, 0x95, + 0x38, 0x58, 0xac, 0x4f, 0xb7, 0x6f, 0x29, 0xf7, 0x39, 0xd0, 0x27, 0x95, 0xfb, 0x94, 0xb4, 0x3f, + 0x43, 0x7c, 0xb2, 0x1c, 0xde, 0xaf, 0x5b, 0xad, 0xd0, 0x1c, 0x7a, 0xfa, 0x96, 0xa1, 0x39, 0xf4, + 0x75, 0x28, 0xa3, 0x73, 0x48, 0xfb, 0x56, 0xe1, 0x39, 0xf4, 0x76, 0x01, 0xc3, 0x73, 0xe8, 0x6b, + 0x7e, 0xc5, 0xee, 0x27, 0x5d, 0x5e, 0xc4, 0x7e, 0x7a, 0x57, 0xb8, 0x12, 0x07, 0x8b, 0xf5, 0xe9, + 0xf6, 0x9d, 0xe4, 0x3e, 0x07, 0x5a, 0x5b, 0x72, 0x9f, 0x83, 0xed, 0xab, 0x30, 0x9f, 0x4f, 0x60, + 0xd6, 0x7b, 0x85, 0x47, 0x57, 0x86, 0xec, 0x3b, 0x28, 0xc5, 0x78, 0x60, 0xb4, 0xe7, 0xcf, 0x21, + 0xeb, 0x6b, 0xfc, 0x21, 0xe9, 0x8c, 0xb2, 0x46, 0xa3, 0xb2, 0x3a, 0x04, 0x32, 0xd6, 0xb9, 0xaf, + 0xa7, 0x25, 0x77, 0x2e, 0xeb, 0xdb, 0xc9, 0x9d, 0x4b, 0x1b, 0x64, 0x11, 0xce, 0x7d, 0xad, 0x2b, + 0xb9, 0x73, 0x59, 0x8f, 0x4c, 0xee, 0x5c, 0xde, 0x07, 0x8b, 0x24, 0x19, 0xbf, 0x0a, 0x86, 0x92, + 0xcc, 0xdf, 0x3e, 0x08, 0x25, 0x59, 0xb0, 0x17, 0x10, 0x4d, 0x32, 0x71, 0x6f, 0x0d, 0x27, 0x59, + 0xe0, 0xb2, 0x1d, 0x4e, 0xb2, 0xe0, 0x15, 0x38, 0x96, 0x64, 0x62, 0xc1, 0x11, 0x24, 0x0b, 0xac, + 0x79, 0x75, 0x08, 0xe4, 0x90, 0x79, 0x8e, 0x74, 0x2e, 0xeb, 0xd7, 0x44, 0xe5, 0x79, 0x48, 0xe7, + 0x2c, 0xcf, 0xbc, 0x70, 0x0f, 0xcd, 0xb3, 0xff, 0x4a, 0x14, 0x9a, 0xe7, 0xc0, 0xad, 0x21, 0x26, + 0xcf, 0xe2, 0x4e, 0x19, 0x9e, 0xe7, 0xc0, 0x45, 0x38, 0x3c, 0xcf, 0xc1, 0xeb, 0x69, 0xec, 0x79, + 0x16, 0x0b, 0x8e, 0x38, 0xcf, 0x81, 0x35, 0xaf, 0x0e, 0x81, 0x8c, 0xfd, 0x38, 0x39, 0xb7, 0x19, + 0xf9, 0xc7, 0x29, 0x78, 0x57, 0x52, 0x2e, 0xc7, 0xa0, 0x62, 0xf7, 0xd9, 0x7b, 0x75, 0x90, 0xef, + 0xb3, 0xe4, 0x5a, 0xa4, 0x14, 0xe3, 0x81, 0xd1, 0x9e, 0x0f, 0x20, 0xe3, 0x29, 0x80, 0xd1, 0xca, + 0x70, 0x35, 0xbb, 0x72, 0x25, 0x16, 0x17, 0xbb, 0x60, 0x6f, 0x7d, 0x2b, 0x5f, 0xb0, 0xa4, 0x98, + 0x56, 0x8a, 0xf1, 0xc0, 0x58, 0xcf, 0xde, 0x5a, 0x56, 0xee, 0x59, 0x52, 0x2f, 0x2b, 0xc5, 0x78, + 0x60, 0xa4, 0xe7, 0xca, 0x85, 0xa7, 0xcf, 0x0a, 0x67, 0x7e, 0x7f, 0x56, 0x38, 0xf3, 0xcf, 0xb3, + 0x42, 0xe2, 0x8b, 0x7e, 0x21, 0xf1, 0xb4, 0x5f, 0x48, 0xfc, 0xd6, 0x2f, 0x24, 0xfe, 0xe8, 0x17, + 0x12, 0xbb, 0x53, 0xf4, 0xbf, 0x51, 0xae, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xe8, 0xa4, + 0xf9, 0x06, 0x23, 0x00, 0x00, } diff --git a/api/control.proto b/api/control.proto index a05c55b4ad..e0b37a9156 100644 --- a/api/control.proto +++ b/api/control.proto @@ -313,12 +313,17 @@ message ListClustersResponse { repeated Cluster clusters = 1; } -message JoinTokenRotation { - // RotateWorkerToken tells UpdateCluster to rotate the worker secret. - bool rotate_worker_token = 1; +// KeyRotation tells UpdateCluster what items to rotate +message KeyRotation { + // WorkerJoinToken tells UpdateCluster to rotate the worker secret token. + bool worker_join_token = 1; + + // ManagerJoinToken tells UpdateCluster to rotate the manager secret token. + bool manager_join_token = 2; + + // ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key + bool manager_unlock_key = 3; - // RotateManagerSecret tells UpdateCluster to rotate the manager secret. - bool rotate_manager_token = 2; } message UpdateClusterRequest { @@ -331,8 +336,8 @@ message UpdateClusterRequest { // Spec is the new spec to apply to the cluster. ClusterSpec spec = 3; - // Rotation contains flags for join token rotation - JoinTokenRotation rotation = 4 [(gogoproto.nullable) = false]; + // Rotation contains flags for join token and unlock key rotation + KeyRotation rotation = 4 [(gogoproto.nullable) = false]; } message UpdateClusterResponse { diff --git a/api/objects.pb.go b/api/objects.pb.go index fc54f1b74b..1e6cb0da09 100644 --- a/api/objects.pb.go +++ b/api/objects.pb.go @@ -232,6 +232,12 @@ type Cluster struct { // be honored. It's a mapping from CN -> BlacklistedCertificate. // swarm. Their certificates should effectively be blacklisted. BlacklistedCertificates map[string]*BlacklistedCertificate `protobuf:"bytes,8,rep,name=blacklisted_certificates,json=blacklistedCertificates" json:"blacklisted_certificates,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` + // UnlockKeys defines the keys that lock node data at rest. For example, + // this would contain the key encrypting key (KEK) that will encrypt the + // manager TLS keys at rest and the raft encryption keys at rest. + // If the key is empty, the node will be unlocked (will not require a key + // to start up from a shut down state). + UnlockKeys []*EncryptionKey `protobuf:"bytes,9,rep,name=unlock_keys,json=unlockKeys" json:"unlock_keys,omitempty"` } func (m *Cluster) Reset() { *m = Cluster{} } @@ -460,6 +466,13 @@ func (m *Cluster) Copy() *Cluster { } } + if m.UnlockKeys != nil { + o.UnlockKeys = make([]*EncryptionKey, 0, len(m.UnlockKeys)) + for _, v := range m.UnlockKeys { + o.UnlockKeys = append(o.UnlockKeys, v.Copy()) + } + } + return o } @@ -633,7 +646,7 @@ func (this *Cluster) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&api.Cluster{") s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n") s = append(s, "Meta: "+strings.Replace(this.Meta.GoString(), `&`, ``, 1)+",\n") @@ -656,6 +669,9 @@ func (this *Cluster) GoString() string { if this.BlacklistedCertificates != nil { s = append(s, "BlacklistedCertificates: "+mapStringForBlacklistedCertificates+",\n") } + if this.UnlockKeys != nil { + s = append(s, "UnlockKeys: "+fmt.Sprintf("%#v", this.UnlockKeys)+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -1310,6 +1326,18 @@ func (m *Cluster) MarshalTo(data []byte) (int, error) { } } } + if len(m.UnlockKeys) > 0 { + for _, msg := range m.UnlockKeys { + data[i] = 0x4a + i++ + i = encodeVarintObjects(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } return i, nil } @@ -1637,6 +1665,12 @@ func (m *Cluster) Size() (n int) { n += mapEntrySize + 1 + sovObjects(uint64(mapEntrySize)) } } + if len(m.UnlockKeys) > 0 { + for _, e := range m.UnlockKeys { + l = e.Size() + n += 1 + l + sovObjects(uint64(l)) + } + } return n } @@ -1814,6 +1848,7 @@ func (this *Cluster) String() string { `NetworkBootstrapKeys:` + strings.Replace(fmt.Sprintf("%v", this.NetworkBootstrapKeys), "EncryptionKey", "EncryptionKey", 1) + `,`, `EncryptionKeyLamportClock:` + fmt.Sprintf("%v", this.EncryptionKeyLamportClock) + `,`, `BlacklistedCertificates:` + mapStringForBlacklistedCertificates + `,`, + `UnlockKeys:` + strings.Replace(fmt.Sprintf("%v", this.UnlockKeys), "EncryptionKey", "EncryptionKey", 1) + `,`, `}`, }, "") return s @@ -3863,6 +3898,37 @@ func (m *Cluster) Unmarshal(data []byte) error { m.BlacklistedCertificates[mapkey] = mapvalue } iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UnlockKeys", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthObjects + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UnlockKeys = append(m.UnlockKeys, &EncryptionKey{}) + if err := m.UnlockKeys[len(m.UnlockKeys)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipObjects(data[iNdEx:]) @@ -4199,79 +4265,80 @@ var ( func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) } var fileDescriptorObjects = []byte{ - // 1174 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x8f, 0x1b, 0x35, - 0x18, 0xee, 0x24, 0xb3, 0xf9, 0x78, 0xb3, 0x59, 0x81, 0xa9, 0xca, 0x34, 0x2c, 0xc9, 0x92, 0x0a, - 0x54, 0xa1, 0x2a, 0x15, 0xa5, 0xa0, 0x2d, 0xb4, 0x82, 0x7c, 0x09, 0xa2, 0x52, 0xa8, 0xdc, 0xd2, - 0x1e, 0x23, 0xef, 0x8c, 0x1b, 0x86, 0x4c, 0xc6, 0x23, 0xdb, 0x49, 0x95, 0x9e, 0x10, 0x3f, 0x80, - 0x9f, 0xc0, 0x5f, 0xe1, 0xba, 0x07, 0x0e, 0xdc, 0xe0, 0x80, 0x22, 0x36, 0x07, 0x24, 0x6e, 0xfc, - 0x04, 0x64, 0x8f, 0x27, 0x99, 0x55, 0x26, 0xcb, 0x56, 0xaa, 0xf6, 0xe6, 0x37, 0x7e, 0x9e, 0xc7, - 0xef, 0x97, 0xdf, 0x71, 0xa0, 0xca, 0x8e, 0xbe, 0xa7, 0xae, 0x14, 0xad, 0x88, 0x33, 0xc9, 0x10, - 0xf2, 0x98, 0x3b, 0xa6, 0xbc, 0x25, 0x9e, 0x13, 0x3e, 0x19, 0xfb, 0xb2, 0x35, 0xfb, 0xa0, 0x56, - 0x91, 0xf3, 0x88, 0x1a, 0x40, 0xad, 0x22, 0x22, 0xea, 0x26, 0xc6, 0x55, 0xe9, 0x4f, 0xa8, 0x90, - 0x64, 0x12, 0xdd, 0x5c, 0xad, 0xcc, 0xd6, 0xe5, 0x11, 0x1b, 0x31, 0xbd, 0xbc, 0xa9, 0x56, 0xf1, - 0xaf, 0xcd, 0x5f, 0x2c, 0xb0, 0x1f, 0x50, 0x49, 0xd0, 0xa7, 0x50, 0x9c, 0x51, 0x2e, 0x7c, 0x16, - 0x3a, 0xd6, 0x81, 0x75, 0xbd, 0x72, 0xeb, 0xad, 0xd6, 0xe6, 0xc9, 0xad, 0x27, 0x31, 0xa4, 0x63, - 0x1f, 0x2f, 0x1a, 0x97, 0x70, 0xc2, 0x40, 0x77, 0x01, 0x5c, 0x4e, 0x89, 0xa4, 0xde, 0x90, 0x48, - 0x27, 0xa7, 0xf9, 0x6f, 0x67, 0xf1, 0x1f, 0x27, 0x4e, 0xe1, 0xb2, 0x21, 0xb4, 0xa5, 0x62, 0x4f, - 0x23, 0x2f, 0x61, 0xe7, 0xcf, 0xc5, 0x36, 0x84, 0xb6, 0x6c, 0xfe, 0x93, 0x07, 0xfb, 0x6b, 0xe6, - 0x51, 0x74, 0x05, 0x72, 0xbe, 0xa7, 0x9d, 0x2f, 0x77, 0x0a, 0xcb, 0x45, 0x23, 0x37, 0xe8, 0xe1, - 0x9c, 0xef, 0xa1, 0x5b, 0x60, 0x4f, 0xa8, 0x24, 0xc6, 0x2d, 0x27, 0x4b, 0x58, 0x65, 0xc0, 0xc4, - 0xa4, 0xb1, 0xe8, 0x63, 0xb0, 0x55, 0x5a, 0x8d, 0x33, 0xfb, 0x59, 0x1c, 0x75, 0xe6, 0xa3, 0x88, - 0xba, 0x09, 0x4f, 0xe1, 0x51, 0x1f, 0x2a, 0x1e, 0x15, 0x2e, 0xf7, 0x23, 0xa9, 0x32, 0x69, 0x6b, - 0xfa, 0xb5, 0x6d, 0xf4, 0xde, 0x1a, 0x8a, 0xd3, 0x3c, 0x74, 0x17, 0x0a, 0x42, 0x12, 0x39, 0x15, - 0xce, 0x8e, 0x56, 0xa8, 0x6f, 0x75, 0x40, 0xa3, 0x8c, 0x0b, 0x86, 0x83, 0xbe, 0x84, 0xbd, 0x09, - 0x09, 0xc9, 0x88, 0xf2, 0xa1, 0x51, 0x29, 0x68, 0x95, 0x77, 0x32, 0x43, 0x8f, 0x91, 0xb1, 0x10, - 0xae, 0x4e, 0xd2, 0x26, 0xea, 0x03, 0x10, 0x29, 0x89, 0xfb, 0xdd, 0x84, 0x86, 0xd2, 0x29, 0x6a, - 0x95, 0x77, 0x33, 0x7d, 0xa1, 0xf2, 0x39, 0xe3, 0xe3, 0xf6, 0x0a, 0x8c, 0x53, 0x44, 0xf4, 0x05, - 0x54, 0x5c, 0xca, 0xa5, 0xff, 0xcc, 0x77, 0x89, 0xa4, 0x4e, 0x49, 0xeb, 0x34, 0xb2, 0x74, 0xba, - 0x6b, 0x98, 0x09, 0x2a, 0xcd, 0x6c, 0xfe, 0x9e, 0x83, 0xe2, 0x23, 0xca, 0x67, 0xbe, 0xfb, 0x6a, - 0xcb, 0x7d, 0xe7, 0x54, 0xb9, 0x33, 0x3d, 0x33, 0xc7, 0x6e, 0x54, 0xfc, 0x10, 0x4a, 0x34, 0xf4, - 0x22, 0xe6, 0x87, 0xd2, 0x94, 0x3b, 0xb3, 0x5b, 0xfa, 0x06, 0x83, 0x57, 0x68, 0xd4, 0x87, 0x6a, - 0xdc, 0xc5, 0xc3, 0x53, 0xb5, 0x3e, 0xc8, 0xa2, 0x7f, 0xab, 0x81, 0xa6, 0x48, 0xbb, 0xd3, 0x94, - 0x85, 0x7a, 0x50, 0x8d, 0x38, 0x9d, 0xf9, 0x6c, 0x2a, 0x86, 0x3a, 0x88, 0xc2, 0xb9, 0x82, 0xc0, - 0xbb, 0x09, 0x4b, 0x59, 0xcd, 0x9f, 0x73, 0x50, 0x4a, 0x7c, 0x44, 0xb7, 0x4d, 0x3a, 0xac, 0xed, - 0x0e, 0x25, 0x58, 0x2d, 0x15, 0x67, 0xe2, 0x36, 0xec, 0x44, 0x8c, 0x4b, 0xe1, 0xe4, 0x0e, 0xf2, - 0xdb, 0x7a, 0xf6, 0x21, 0xe3, 0xb2, 0xcb, 0xc2, 0x67, 0xfe, 0x08, 0xc7, 0x60, 0xf4, 0x14, 0x2a, - 0x33, 0x9f, 0xcb, 0x29, 0x09, 0x86, 0x7e, 0x24, 0x9c, 0xbc, 0xe6, 0xbe, 0x77, 0xd6, 0x91, 0xad, - 0x27, 0x31, 0x7e, 0xf0, 0xb0, 0xb3, 0xb7, 0x5c, 0x34, 0x60, 0x65, 0x0a, 0x0c, 0x46, 0x6a, 0x10, - 0x89, 0xda, 0x03, 0x28, 0xaf, 0x76, 0xd0, 0x0d, 0x80, 0x30, 0x6e, 0xd1, 0xe1, 0xaa, 0x69, 0xaa, - 0xcb, 0x45, 0xa3, 0x6c, 0x1a, 0x77, 0xd0, 0xc3, 0x65, 0x03, 0x18, 0x78, 0x08, 0x81, 0x4d, 0x3c, - 0x8f, 0xeb, 0x16, 0x2a, 0x63, 0xbd, 0x6e, 0xfe, 0xba, 0x03, 0xf6, 0x63, 0x22, 0xc6, 0x17, 0x3d, - 0x66, 0xd4, 0x99, 0x1b, 0x4d, 0x77, 0x03, 0x40, 0xc4, 0xa5, 0x54, 0xe1, 0xd8, 0xeb, 0x70, 0x4c, - 0x81, 0x55, 0x38, 0x06, 0x10, 0x87, 0x23, 0x02, 0x26, 0x75, 0x7f, 0xd9, 0x58, 0xaf, 0xd1, 0x35, - 0x28, 0x86, 0xcc, 0xd3, 0xf4, 0x82, 0xa6, 0xc3, 0x72, 0xd1, 0x28, 0xa8, 0x91, 0x32, 0xe8, 0xe1, - 0x82, 0xda, 0x1a, 0x78, 0xea, 0xde, 0x92, 0x30, 0x64, 0x92, 0xa8, 0xa1, 0x24, 0xcc, 0xfd, 0xcf, - 0x6c, 0xac, 0xf6, 0x1a, 0x96, 0xdc, 0xdb, 0x14, 0x13, 0x3d, 0x81, 0x37, 0x12, 0x7f, 0xd3, 0x82, - 0xa5, 0x97, 0x11, 0x44, 0x46, 0x21, 0xb5, 0x93, 0x9a, 0x93, 0xe5, 0xed, 0x73, 0x52, 0x67, 0x30, - 0x6b, 0x4e, 0x76, 0xa0, 0xea, 0x51, 0xe1, 0x73, 0xea, 0xe9, 0x1b, 0x48, 0x1d, 0x38, 0xb0, 0xae, - 0xef, 0x6d, 0xf9, 0xf4, 0x18, 0x11, 0x8a, 0x77, 0x0d, 0x47, 0x5b, 0xa8, 0x0d, 0x25, 0xd3, 0x37, - 0xc2, 0xa9, 0xe8, 0xde, 0x3d, 0xe7, 0x7c, 0x5c, 0xd1, 0x4e, 0x4d, 0x90, 0xdd, 0x97, 0x9a, 0x20, - 0x77, 0x00, 0x02, 0x36, 0x1a, 0x7a, 0xdc, 0x9f, 0x51, 0xee, 0x54, 0x35, 0xb7, 0x96, 0xc5, 0xed, - 0x69, 0x04, 0x2e, 0x07, 0x6c, 0x14, 0x2f, 0x9b, 0x3f, 0x5a, 0xf0, 0xfa, 0x86, 0x53, 0xe8, 0x23, - 0x28, 0x1a, 0xb7, 0xce, 0x7a, 0x04, 0x18, 0x1e, 0x4e, 0xb0, 0x68, 0x1f, 0xca, 0xea, 0x8e, 0x50, - 0x21, 0x68, 0x7c, 0xfb, 0xcb, 0x78, 0xfd, 0x03, 0x72, 0xa0, 0x48, 0x02, 0x9f, 0xa8, 0xbd, 0xbc, - 0xde, 0x4b, 0xcc, 0xe6, 0x4f, 0x39, 0x28, 0x1a, 0xb1, 0x8b, 0x1e, 0xe7, 0xe6, 0xd8, 0x8d, 0x9b, - 0x75, 0x0f, 0x76, 0xe3, 0x74, 0x9a, 0x96, 0xb0, 0xff, 0x37, 0xa9, 0x95, 0x18, 0x1f, 0xb7, 0xc3, - 0x3d, 0xb0, 0xfd, 0x88, 0x4c, 0xcc, 0x28, 0xcf, 0x3c, 0x79, 0xf0, 0xb0, 0xfd, 0xe0, 0x9b, 0x28, - 0xee, 0xec, 0xd2, 0x72, 0xd1, 0xb0, 0xd5, 0x0f, 0x58, 0xd3, 0x9a, 0x7f, 0xda, 0x50, 0xec, 0x06, - 0x53, 0x21, 0x29, 0xbf, 0xe8, 0x84, 0x98, 0x63, 0x37, 0x12, 0xd2, 0x85, 0x22, 0x67, 0x4c, 0x0e, - 0x5d, 0x72, 0x56, 0x2e, 0x30, 0x63, 0xb2, 0xdb, 0xee, 0xec, 0x29, 0xa2, 0x1a, 0x24, 0xb1, 0x8d, - 0x0b, 0x8a, 0xda, 0x25, 0xe8, 0x29, 0x5c, 0x49, 0xc6, 0xef, 0x11, 0x63, 0x52, 0x48, 0x4e, 0xa2, - 0xe1, 0x98, 0xce, 0xd5, 0x37, 0x2f, 0xbf, 0xed, 0x65, 0xd2, 0x0f, 0x5d, 0x3e, 0xd7, 0x89, 0xba, - 0x4f, 0xe7, 0xf8, 0xb2, 0x11, 0xe8, 0x24, 0xfc, 0xfb, 0x74, 0x2e, 0xd0, 0x67, 0xb0, 0x4f, 0x57, - 0x30, 0xa5, 0x38, 0x0c, 0xc8, 0x44, 0x7d, 0x58, 0x86, 0x6e, 0xc0, 0xdc, 0xb1, 0x9e, 0x6d, 0x36, - 0xbe, 0x4a, 0xd3, 0x52, 0x5f, 0xc5, 0x88, 0xae, 0x02, 0x20, 0x01, 0xce, 0x51, 0x40, 0xdc, 0x71, - 0xe0, 0x0b, 0xf5, 0xfe, 0x4c, 0x3d, 0x36, 0xd4, 0x78, 0x52, 0xbe, 0x1d, 0x9e, 0x91, 0xad, 0x56, - 0x67, 0xcd, 0x4d, 0x3d, 0x5d, 0x44, 0x3f, 0x94, 0x7c, 0x8e, 0xdf, 0x3c, 0xca, 0xde, 0xad, 0xcd, - 0x60, 0xff, 0x2c, 0x22, 0x7a, 0x0d, 0xf2, 0x63, 0x3a, 0x8f, 0x6b, 0x8f, 0xd5, 0x12, 0x7d, 0x0e, - 0x3b, 0x33, 0x12, 0x4c, 0xa9, 0xa9, 0xfa, 0xfb, 0x59, 0x3e, 0x65, 0x4b, 0xe2, 0x98, 0xf8, 0x49, - 0xee, 0xd0, 0x6a, 0xfe, 0x6d, 0x41, 0xe1, 0x11, 0x75, 0x39, 0x95, 0xaf, 0xb4, 0xbb, 0x0e, 0x4f, - 0x75, 0x57, 0x3d, 0xfb, 0xe1, 0xa1, 0x4e, 0xdd, 0x68, 0xae, 0x2b, 0x50, 0xf0, 0xfc, 0x11, 0x15, - 0xf1, 0xd3, 0xa9, 0x8c, 0x8d, 0x85, 0x9a, 0x60, 0x0b, 0xff, 0x05, 0xd5, 0xd7, 0x28, 0x1f, 0x7f, - 0xe5, 0x8d, 0x82, 0xff, 0x82, 0x62, 0xbd, 0x87, 0x6a, 0x50, 0xf2, 0x43, 0x49, 0x79, 0x48, 0x02, - 0x5d, 0xe6, 0x12, 0x5e, 0xd9, 0x9d, 0xfd, 0xe3, 0x93, 0xfa, 0xa5, 0x3f, 0x4e, 0xea, 0x97, 0xfe, - 0x3d, 0xa9, 0x5b, 0x3f, 0x2c, 0xeb, 0xd6, 0xf1, 0xb2, 0x6e, 0xfd, 0xb6, 0xac, 0x5b, 0x7f, 0x2d, - 0xeb, 0xd6, 0x51, 0x41, 0xff, 0xf5, 0xf9, 0xf0, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x40, 0xcf, - 0x57, 0x63, 0x6a, 0x0d, 0x00, 0x00, + // 1192 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0xef, 0xda, 0x1b, 0xdb, 0xfb, 0x1c, 0x47, 0x30, 0x54, 0x65, 0x1b, 0x82, 0x1d, 0x5c, 0x81, + 0x2a, 0x54, 0xb9, 0xa2, 0x14, 0x94, 0x42, 0x2b, 0xb0, 0x9d, 0x08, 0xac, 0x52, 0xa8, 0xa6, 0xa5, + 0x3d, 0x5a, 0x93, 0xdd, 0xa9, 0x59, 0xbc, 0xde, 0x59, 0xcd, 0x8c, 0x5d, 0xb9, 0x27, 0xc4, 0x07, + 0xe0, 0x23, 0x20, 0xbe, 0x09, 0xd7, 0x1e, 0x38, 0x70, 0x83, 0x53, 0x44, 0x7d, 0x40, 0xe2, 0xc6, + 0x47, 0x40, 0xf3, 0x67, 0x9d, 0x8d, 0xbc, 0x0e, 0xa9, 0x54, 0xe5, 0x36, 0xe3, 0xf9, 0xfd, 0x7e, + 0xef, 0xcd, 0x9b, 0xdf, 0xbc, 0x1d, 0x43, 0x83, 0x1d, 0x7e, 0x4f, 0x03, 0x29, 0x3a, 0x29, 0x67, + 0x92, 0x21, 0x14, 0xb2, 0x60, 0x4c, 0x79, 0x47, 0x3c, 0x25, 0x7c, 0x32, 0x8e, 0x64, 0x67, 0xf6, + 0xc1, 0x76, 0x5d, 0xce, 0x53, 0x6a, 0x01, 0xdb, 0x75, 0x91, 0xd2, 0x20, 0x9b, 0x5c, 0x96, 0xd1, + 0x84, 0x0a, 0x49, 0x26, 0xe9, 0xf5, 0xe5, 0xc8, 0x2e, 0x5d, 0x1c, 0xb1, 0x11, 0xd3, 0xc3, 0xeb, + 0x6a, 0x64, 0x7e, 0x6d, 0xff, 0xea, 0x80, 0x7b, 0x8f, 0x4a, 0x82, 0x3e, 0x85, 0xea, 0x8c, 0x72, + 0x11, 0xb1, 0xc4, 0x77, 0x76, 0x9d, 0xab, 0xf5, 0x1b, 0x6f, 0x75, 0x56, 0x23, 0x77, 0x1e, 0x19, + 0x48, 0xcf, 0x7d, 0x7e, 0xd4, 0xba, 0x80, 0x33, 0x06, 0xba, 0x0d, 0x10, 0x70, 0x4a, 0x24, 0x0d, + 0x87, 0x44, 0xfa, 0x25, 0xcd, 0x7f, 0xbb, 0x88, 0xff, 0x30, 0x4b, 0x0a, 0x7b, 0x96, 0xd0, 0x95, + 0x8a, 0x3d, 0x4d, 0xc3, 0x8c, 0x5d, 0x3e, 0x13, 0xdb, 0x12, 0xba, 0xb2, 0xfd, 0x4f, 0x19, 0xdc, + 0xaf, 0x59, 0x48, 0xd1, 0x25, 0x28, 0x45, 0xa1, 0x4e, 0xde, 0xeb, 0x55, 0x16, 0x47, 0xad, 0xd2, + 0x60, 0x1f, 0x97, 0xa2, 0x10, 0xdd, 0x00, 0x77, 0x42, 0x25, 0xb1, 0x69, 0xf9, 0x45, 0xc2, 0xaa, + 0x02, 0x76, 0x4f, 0x1a, 0x8b, 0x3e, 0x06, 0x57, 0x95, 0xd5, 0x26, 0xb3, 0x53, 0xc4, 0x51, 0x31, + 0x1f, 0xa4, 0x34, 0xc8, 0x78, 0x0a, 0x8f, 0x0e, 0xa0, 0x1e, 0x52, 0x11, 0xf0, 0x28, 0x95, 0xaa, + 0x92, 0xae, 0xa6, 0x5f, 0x59, 0x47, 0xdf, 0x3f, 0x86, 0xe2, 0x3c, 0x0f, 0xdd, 0x86, 0x8a, 0x90, + 0x44, 0x4e, 0x85, 0xbf, 0xa1, 0x15, 0x9a, 0x6b, 0x13, 0xd0, 0x28, 0x9b, 0x82, 0xe5, 0xa0, 0x2f, + 0x61, 0x6b, 0x42, 0x12, 0x32, 0xa2, 0x7c, 0x68, 0x55, 0x2a, 0x5a, 0xe5, 0x9d, 0xc2, 0xad, 0x1b, + 0xa4, 0x11, 0xc2, 0x8d, 0x49, 0x7e, 0x8a, 0x0e, 0x00, 0x88, 0x94, 0x24, 0xf8, 0x6e, 0x42, 0x13, + 0xe9, 0x57, 0xb5, 0xca, 0xbb, 0x85, 0xb9, 0x50, 0xf9, 0x94, 0xf1, 0x71, 0x77, 0x09, 0xc6, 0x39, + 0x22, 0xfa, 0x02, 0xea, 0x01, 0xe5, 0x32, 0x7a, 0x12, 0x05, 0x44, 0x52, 0xbf, 0xa6, 0x75, 0x5a, + 0x45, 0x3a, 0xfd, 0x63, 0x98, 0xdd, 0x54, 0x9e, 0xd9, 0xfe, 0xa3, 0x04, 0xd5, 0x07, 0x94, 0xcf, + 0xa2, 0xe0, 0xd5, 0x1e, 0xf7, 0xad, 0x13, 0xc7, 0x5d, 0x98, 0x99, 0x0d, 0xbb, 0x72, 0xe2, 0x7b, + 0x50, 0xa3, 0x49, 0x98, 0xb2, 0x28, 0x91, 0xf6, 0xb8, 0x0b, 0xdd, 0x72, 0x60, 0x31, 0x78, 0x89, + 0x46, 0x07, 0xd0, 0x30, 0x2e, 0x1e, 0x9e, 0x38, 0xeb, 0xdd, 0x22, 0xfa, 0xb7, 0x1a, 0x68, 0x0f, + 0x69, 0x73, 0x9a, 0x9b, 0xa1, 0x7d, 0x68, 0xa4, 0x9c, 0xce, 0x22, 0x36, 0x15, 0x43, 0xbd, 0x89, + 0xca, 0x99, 0x36, 0x81, 0x37, 0x33, 0x96, 0x9a, 0xb5, 0x7f, 0x2e, 0x41, 0x2d, 0xcb, 0x11, 0xdd, + 0xb4, 0xe5, 0x70, 0xd6, 0x27, 0x94, 0x61, 0xb5, 0x94, 0xa9, 0xc4, 0x4d, 0xd8, 0x48, 0x19, 0x97, + 0xc2, 0x2f, 0xed, 0x96, 0xd7, 0x79, 0xf6, 0x3e, 0xe3, 0xb2, 0xcf, 0x92, 0x27, 0xd1, 0x08, 0x1b, + 0x30, 0x7a, 0x0c, 0xf5, 0x59, 0xc4, 0xe5, 0x94, 0xc4, 0xc3, 0x28, 0x15, 0x7e, 0x59, 0x73, 0xdf, + 0x3b, 0x2d, 0x64, 0xe7, 0x91, 0xc1, 0x0f, 0xee, 0xf7, 0xb6, 0x16, 0x47, 0x2d, 0x58, 0x4e, 0x05, + 0x06, 0x2b, 0x35, 0x48, 0xc5, 0xf6, 0x3d, 0xf0, 0x96, 0x2b, 0xe8, 0x1a, 0x40, 0x62, 0x2c, 0x3a, + 0x5c, 0x9a, 0xa6, 0xb1, 0x38, 0x6a, 0x79, 0xd6, 0xb8, 0x83, 0x7d, 0xec, 0x59, 0xc0, 0x20, 0x44, + 0x08, 0x5c, 0x12, 0x86, 0x5c, 0x5b, 0xc8, 0xc3, 0x7a, 0xdc, 0xfe, 0x6d, 0x03, 0xdc, 0x87, 0x44, + 0x8c, 0xcf, 0xbb, 0xcd, 0xa8, 0x98, 0x2b, 0xa6, 0xbb, 0x06, 0x20, 0xcc, 0x51, 0xaa, 0xed, 0xb8, + 0xc7, 0xdb, 0xb1, 0x07, 0xac, 0xb6, 0x63, 0x01, 0x66, 0x3b, 0x22, 0x66, 0x52, 0xfb, 0xcb, 0xc5, + 0x7a, 0x8c, 0xae, 0x40, 0x35, 0x61, 0xa1, 0xa6, 0x57, 0x34, 0x1d, 0x16, 0x47, 0xad, 0x8a, 0x6a, + 0x29, 0x83, 0x7d, 0x5c, 0x51, 0x4b, 0x83, 0x50, 0xdd, 0x5b, 0x92, 0x24, 0x4c, 0x12, 0xd5, 0x94, + 0x84, 0xbd, 0xff, 0x85, 0xc6, 0xea, 0x1e, 0xc3, 0xb2, 0x7b, 0x9b, 0x63, 0xa2, 0x47, 0xf0, 0x46, + 0x96, 0x6f, 0x5e, 0xb0, 0xf6, 0x32, 0x82, 0xc8, 0x2a, 0xe4, 0x56, 0x72, 0x7d, 0xd2, 0x5b, 0xdf, + 0x27, 0x75, 0x05, 0x8b, 0xfa, 0x64, 0x0f, 0x1a, 0x21, 0x15, 0x11, 0xa7, 0xa1, 0xbe, 0x81, 0xd4, + 0x87, 0x5d, 0xe7, 0xea, 0xd6, 0x9a, 0x4f, 0x8f, 0x15, 0xa1, 0x78, 0xd3, 0x72, 0xf4, 0x0c, 0x75, + 0xa1, 0x66, 0x7d, 0x23, 0xfc, 0xba, 0xf6, 0xee, 0x19, 0xfb, 0xe3, 0x92, 0x76, 0xa2, 0x83, 0x6c, + 0xbe, 0x54, 0x07, 0xb9, 0x05, 0x10, 0xb3, 0xd1, 0x30, 0xe4, 0xd1, 0x8c, 0x72, 0xbf, 0xa1, 0xb9, + 0xdb, 0x45, 0xdc, 0x7d, 0x8d, 0xc0, 0x5e, 0xcc, 0x46, 0x66, 0xd8, 0xfe, 0xd1, 0x81, 0xd7, 0x57, + 0x92, 0x42, 0x1f, 0x41, 0xd5, 0xa6, 0x75, 0xda, 0x23, 0xc0, 0xf2, 0x70, 0x86, 0x45, 0x3b, 0xe0, + 0xa9, 0x3b, 0x42, 0x85, 0xa0, 0xe6, 0xf6, 0x7b, 0xf8, 0xf8, 0x07, 0xe4, 0x43, 0x95, 0xc4, 0x11, + 0x51, 0x6b, 0x65, 0xbd, 0x96, 0x4d, 0xdb, 0x3f, 0x95, 0xa0, 0x6a, 0xc5, 0xce, 0xbb, 0x9d, 0xdb, + 0xb0, 0x2b, 0x37, 0xeb, 0x0e, 0x6c, 0x9a, 0x72, 0x5a, 0x4b, 0xb8, 0xff, 0x5b, 0xd4, 0xba, 0xc1, + 0x1b, 0x3b, 0xdc, 0x01, 0x37, 0x4a, 0xc9, 0xc4, 0xb6, 0xf2, 0xc2, 0xc8, 0x83, 0xfb, 0xdd, 0x7b, + 0xdf, 0xa4, 0xc6, 0xd9, 0xb5, 0xc5, 0x51, 0xcb, 0x55, 0x3f, 0x60, 0x4d, 0x6b, 0xff, 0xb2, 0x01, + 0xd5, 0x7e, 0x3c, 0x15, 0x92, 0xf2, 0xf3, 0x2e, 0x88, 0x0d, 0xbb, 0x52, 0x90, 0x3e, 0x54, 0x39, + 0x63, 0x72, 0x18, 0x90, 0xd3, 0x6a, 0x81, 0x19, 0x93, 0xfd, 0x6e, 0x6f, 0x4b, 0x11, 0x55, 0x23, + 0x31, 0x73, 0x5c, 0x51, 0xd4, 0x3e, 0x41, 0x8f, 0xe1, 0x52, 0xd6, 0x7e, 0x0f, 0x19, 0x93, 0x42, + 0x72, 0x92, 0x0e, 0xc7, 0x74, 0xae, 0xbe, 0x79, 0xe5, 0x75, 0x2f, 0x93, 0x83, 0x24, 0xe0, 0x73, + 0x5d, 0xa8, 0xbb, 0x74, 0x8e, 0x2f, 0x5a, 0x81, 0x5e, 0xc6, 0xbf, 0x4b, 0xe7, 0x02, 0x7d, 0x06, + 0x3b, 0x74, 0x09, 0x53, 0x8a, 0xc3, 0x98, 0x4c, 0xd4, 0x87, 0x65, 0x18, 0xc4, 0x2c, 0x18, 0xeb, + 0xde, 0xe6, 0xe2, 0xcb, 0x34, 0x2f, 0xf5, 0x95, 0x41, 0xf4, 0x15, 0x00, 0x09, 0xf0, 0x0f, 0x63, + 0x12, 0x8c, 0xe3, 0x48, 0xa8, 0xf7, 0x67, 0xee, 0xb1, 0xa1, 0xda, 0x93, 0xca, 0x6d, 0xef, 0x94, + 0x6a, 0x75, 0x7a, 0xc7, 0xdc, 0xdc, 0xd3, 0x45, 0x1c, 0x24, 0x92, 0xcf, 0xf1, 0x9b, 0x87, 0xc5, + 0xab, 0xa8, 0x07, 0xf5, 0x69, 0xa2, 0xc2, 0x9b, 0x1a, 0x78, 0x67, 0xad, 0x01, 0x18, 0x96, 0xda, + 0xf9, 0xf6, 0x0c, 0x76, 0x4e, 0x0b, 0x8e, 0x5e, 0x83, 0xf2, 0x98, 0xce, 0x8d, 0x7f, 0xb0, 0x1a, + 0xa2, 0xcf, 0x61, 0x63, 0x46, 0xe2, 0x29, 0xb5, 0xce, 0x79, 0xbf, 0x28, 0x5e, 0xb1, 0x24, 0x36, + 0xc4, 0x4f, 0x4a, 0x7b, 0x4e, 0xfb, 0x6f, 0x07, 0x2a, 0x0f, 0x68, 0xc0, 0xa9, 0x7c, 0xa5, 0x0e, + 0xdd, 0x3b, 0xe1, 0xd0, 0x66, 0xf1, 0xe3, 0x45, 0x45, 0x5d, 0x31, 0xe8, 0x25, 0xa8, 0x84, 0xd1, + 0x88, 0x0a, 0xf3, 0xfc, 0xf2, 0xb0, 0x9d, 0xa1, 0x36, 0xb8, 0x22, 0x7a, 0x46, 0xf5, 0x55, 0x2c, + 0x9b, 0x97, 0x82, 0x55, 0x88, 0x9e, 0x51, 0xac, 0xd7, 0xd0, 0x36, 0xd4, 0xa2, 0x44, 0x52, 0x9e, + 0x90, 0x58, 0x5b, 0xa5, 0x86, 0x97, 0xf3, 0xde, 0xce, 0xf3, 0x17, 0xcd, 0x0b, 0x7f, 0xbe, 0x68, + 0x5e, 0xf8, 0xf7, 0x45, 0xd3, 0xf9, 0x61, 0xd1, 0x74, 0x9e, 0x2f, 0x9a, 0xce, 0xef, 0x8b, 0xa6, + 0xf3, 0xd7, 0xa2, 0xe9, 0x1c, 0x56, 0xf4, 0xdf, 0xa7, 0x0f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x49, 0x8f, 0xcd, 0x22, 0xae, 0x0d, 0x00, 0x00, } diff --git a/api/objects.proto b/api/objects.proto index 86421c0851..59f2c267ca 100644 --- a/api/objects.proto +++ b/api/objects.proto @@ -230,6 +230,13 @@ message Cluster { // be honored. It's a mapping from CN -> BlacklistedCertificate. // swarm. Their certificates should effectively be blacklisted. map blacklisted_certificates = 8; + + // UnlockKeys defines the keys that lock node data at rest. For example, + // this would contain the key encrypting key (KEK) that will encrypt the + // manager TLS keys at rest and the raft encryption keys at rest. + // If the key is empty, the node will be unlocked (will not require a key + // to start up from a shut down state). + repeated EncryptionKey unlock_keys = 9; } // Secret represents a secret that should be passed to a container or a node, diff --git a/api/specs.pb.go b/api/specs.pb.go index 97eeca2f8f..30f573e6df 100644 --- a/api/specs.pb.go +++ b/api/specs.pb.go @@ -596,6 +596,8 @@ type ClusterSpec struct { CAConfig CAConfig `protobuf:"bytes,6,opt,name=ca_config,json=caConfig" json:"ca_config"` // TaskDefaults specifies the default values to use for task creation. TaskDefaults TaskDefaults `protobuf:"bytes,7,opt,name=task_defaults,json=taskDefaults" json:"task_defaults"` + // EncryptionConfig defines the cluster's encryption settings. + EncryptionConfig EncryptionConfig `protobuf:"bytes,8,opt,name=encryption_config,json=encryptionConfig" json:"encryption_config"` } func (m *ClusterSpec) Reset() { *m = ClusterSpec{} } @@ -908,6 +910,7 @@ func (m *ClusterSpec) Copy() *ClusterSpec { Dispatcher: *m.Dispatcher.Copy(), CAConfig: *m.CAConfig.Copy(), TaskDefaults: *m.TaskDefaults.Copy(), + EncryptionConfig: *m.EncryptionConfig.Copy(), } return o @@ -1159,7 +1162,7 @@ func (this *ClusterSpec) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&api.ClusterSpec{") s = append(s, "Annotations: "+strings.Replace(this.Annotations.GoString(), `&`, ``, 1)+",\n") s = append(s, "AcceptancePolicy: "+strings.Replace(this.AcceptancePolicy.GoString(), `&`, ``, 1)+",\n") @@ -1168,6 +1171,7 @@ func (this *ClusterSpec) GoString() string { s = append(s, "Dispatcher: "+strings.Replace(this.Dispatcher.GoString(), `&`, ``, 1)+",\n") s = append(s, "CAConfig: "+strings.Replace(this.CAConfig.GoString(), `&`, ``, 1)+",\n") s = append(s, "TaskDefaults: "+strings.Replace(this.TaskDefaults.GoString(), `&`, ``, 1)+",\n") + s = append(s, "EncryptionConfig: "+strings.Replace(this.EncryptionConfig.GoString(), `&`, ``, 1)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2008,6 +2012,14 @@ func (m *ClusterSpec) MarshalTo(data []byte) (int, error) { return 0, err } i += n29 + data[i] = 0x42 + i++ + i = encodeVarintSpecs(data, i, uint64(m.EncryptionConfig.Size())) + n30, err := m.EncryptionConfig.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n30 return i, nil } @@ -2029,11 +2041,11 @@ func (m *SecretSpec) MarshalTo(data []byte) (int, error) { data[i] = 0xa i++ i = encodeVarintSpecs(data, i, uint64(m.Annotations.Size())) - n30, err := m.Annotations.MarshalTo(data[i:]) + n31, err := m.Annotations.MarshalTo(data[i:]) if err != nil { return 0, err } - i += n30 + i += n31 if len(m.Data) > 0 { data[i] = 0x12 i++ @@ -2392,6 +2404,8 @@ func (m *ClusterSpec) Size() (n int) { n += 1 + l + sovSpecs(uint64(l)) l = m.TaskDefaults.Size() n += 1 + l + sovSpecs(uint64(l)) + l = m.EncryptionConfig.Size() + n += 1 + l + sovSpecs(uint64(l)) return n } @@ -2629,6 +2643,7 @@ func (this *ClusterSpec) String() string { `Dispatcher:` + strings.Replace(strings.Replace(this.Dispatcher.String(), "DispatcherConfig", "DispatcherConfig", 1), `&`, ``, 1) + `,`, `CAConfig:` + strings.Replace(strings.Replace(this.CAConfig.String(), "CAConfig", "CAConfig", 1), `&`, ``, 1) + `,`, `TaskDefaults:` + strings.Replace(strings.Replace(this.TaskDefaults.String(), "TaskDefaults", "TaskDefaults", 1), `&`, ``, 1) + `,`, + `EncryptionConfig:` + strings.Replace(strings.Replace(this.EncryptionConfig.String(), "EncryptionConfig", "EncryptionConfig", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -4956,6 +4971,36 @@ func (m *ClusterSpec) Unmarshal(data []byte) error { return err } iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EncryptionConfig", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpecs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSpecs + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.EncryptionConfig.Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSpecs(data[iNdEx:]) @@ -5196,105 +5241,107 @@ var ( func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 1597 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6e, 0xe3, 0xc8, - 0x11, 0x16, 0x2d, 0x59, 0x3f, 0x45, 0x69, 0x46, 0xd3, 0xd8, 0x1f, 0x8e, 0x76, 0x23, 0x69, 0xb4, - 0x93, 0x8d, 0x37, 0x8b, 0x78, 0x12, 0x25, 0xd8, 0xcc, 0x66, 0xb2, 0x48, 0xf4, 0x17, 0x8f, 0xe2, - 0xd8, 0x2b, 0xb4, 0xbd, 0x03, 0xcc, 0x49, 0x68, 0x93, 0x6d, 0x89, 0x30, 0xc5, 0x66, 0x9a, 0x4d, - 0x2d, 0x7c, 0xcb, 0x71, 0x31, 0x87, 0xbc, 0x81, 0x4f, 0x01, 0xf2, 0x06, 0xb9, 0xe4, 0x09, 0xe6, - 0x98, 0x63, 0x4e, 0x46, 0xac, 0x27, 0x08, 0x90, 0x17, 0x08, 0xba, 0xd9, 0x94, 0xa8, 0x2c, 0xbd, - 0x5e, 0x20, 0xbe, 0x75, 0x17, 0xbf, 0xaf, 0xd8, 0x5d, 0xf5, 0xb1, 0xaa, 0x08, 0x66, 0x18, 0x50, - 0x3b, 0xdc, 0x0f, 0x38, 0x13, 0x0c, 0x21, 0x87, 0xd9, 0x17, 0x94, 0xef, 0x87, 0x5f, 0x13, 0xbe, - 0xb8, 0x70, 0xc5, 0xfe, 0xf2, 0x67, 0x0d, 0x53, 0x5c, 0x06, 0x54, 0x03, 0x1a, 0xef, 0xcc, 0xd8, - 0x8c, 0xa9, 0xe5, 0x33, 0xb9, 0xd2, 0xd6, 0xf7, 0x9d, 0x88, 0x13, 0xe1, 0x32, 0xff, 0x59, 0xb2, - 0x88, 0x1f, 0x74, 0xfe, 0x5c, 0x80, 0xf2, 0x31, 0x73, 0xe8, 0x49, 0x40, 0x6d, 0x74, 0x00, 0x26, - 0xf1, 0x7d, 0x26, 0x14, 0x20, 0xb4, 0x8c, 0xb6, 0xb1, 0x67, 0x76, 0x5b, 0xfb, 0xdf, 0x7e, 0xe5, - 0x7e, 0x6f, 0x03, 0xeb, 0x17, 0xde, 0x5e, 0xb7, 0x72, 0x38, 0xcd, 0x44, 0x3f, 0x85, 0x02, 0x67, - 0x1e, 0xb5, 0x76, 0xda, 0xc6, 0xde, 0x83, 0xee, 0x87, 0x59, 0x1e, 0xe4, 0x4b, 0x31, 0xf3, 0x28, - 0x56, 0x48, 0x74, 0x00, 0xb0, 0xa0, 0x8b, 0x33, 0xca, 0xc3, 0xb9, 0x1b, 0x58, 0x79, 0xc5, 0xfb, - 0xd1, 0x6d, 0x3c, 0x79, 0xd8, 0xfd, 0xa3, 0x35, 0x1c, 0xa7, 0xa8, 0xe8, 0x08, 0xaa, 0x64, 0x49, - 0x5c, 0x8f, 0x9c, 0xb9, 0x9e, 0x2b, 0x2e, 0xad, 0x82, 0x72, 0xf5, 0xc9, 0x77, 0xba, 0xea, 0xa5, - 0x08, 0x78, 0x8b, 0xde, 0x71, 0x00, 0x36, 0x2f, 0x42, 0x1f, 0x43, 0x69, 0x32, 0x3a, 0x1e, 0x8e, - 0x8f, 0x0f, 0xea, 0xb9, 0xc6, 0xe3, 0x37, 0x57, 0xed, 0x77, 0xa5, 0x8f, 0x0d, 0x60, 0x42, 0x7d, - 0xc7, 0xf5, 0x67, 0x68, 0x0f, 0xca, 0xbd, 0xc1, 0x60, 0x34, 0x39, 0x1d, 0x0d, 0xeb, 0x46, 0xa3, - 0xf1, 0xe6, 0xaa, 0xfd, 0xde, 0x36, 0xb0, 0x67, 0xdb, 0x34, 0x10, 0xd4, 0x69, 0x14, 0xbe, 0xf9, - 0x4b, 0x33, 0xd7, 0xf9, 0xc6, 0x80, 0x6a, 0xfa, 0x10, 0xe8, 0x63, 0x28, 0xf6, 0x06, 0xa7, 0xe3, - 0x57, 0xa3, 0x7a, 0x6e, 0x43, 0x4f, 0x23, 0x7a, 0xb6, 0x70, 0x97, 0x14, 0x3d, 0x85, 0xdd, 0x49, - 0xef, 0xab, 0x93, 0x51, 0xdd, 0xd8, 0x1c, 0x27, 0x0d, 0x9b, 0x90, 0x28, 0x54, 0xa8, 0x21, 0xee, - 0x8d, 0x8f, 0xeb, 0x3b, 0xd9, 0xa8, 0x21, 0x27, 0xae, 0xaf, 0x8f, 0x72, 0x93, 0x07, 0xf3, 0x84, - 0xf2, 0xa5, 0x6b, 0xdf, 0xb3, 0x26, 0x3e, 0x83, 0x82, 0x20, 0xe1, 0x85, 0xd2, 0x84, 0x99, 0xad, - 0x89, 0x53, 0x12, 0x5e, 0xc8, 0x97, 0x6a, 0xba, 0xc2, 0x4b, 0x65, 0x70, 0x1a, 0x78, 0xae, 0x4d, - 0x04, 0x75, 0x94, 0x32, 0xcc, 0xee, 0x0f, 0xb3, 0xd8, 0x78, 0x8d, 0xd2, 0xe7, 0x7f, 0x99, 0xc3, - 0x29, 0x2a, 0x7a, 0x01, 0xc5, 0x99, 0xc7, 0xce, 0x88, 0xa7, 0x34, 0x61, 0x76, 0x9f, 0x64, 0x39, - 0x39, 0x50, 0x88, 0x8d, 0x03, 0x4d, 0x41, 0xcf, 0xa1, 0x18, 0x05, 0x0e, 0x11, 0xd4, 0x2a, 0x2a, - 0x72, 0x3b, 0x8b, 0xfc, 0x95, 0x42, 0x0c, 0x98, 0x7f, 0xee, 0xce, 0xb0, 0xc6, 0xa3, 0x43, 0x28, - 0xfb, 0x54, 0x7c, 0xcd, 0xf8, 0x45, 0x68, 0x95, 0xda, 0xf9, 0x3d, 0xb3, 0xfb, 0x69, 0xa6, 0x18, - 0x63, 0x4c, 0x4f, 0x08, 0x62, 0xcf, 0x17, 0xd4, 0x17, 0xb1, 0x9b, 0xfe, 0x8e, 0x65, 0xe0, 0xb5, - 0x03, 0xf4, 0x6b, 0x28, 0x53, 0xdf, 0x09, 0x98, 0xeb, 0x0b, 0xab, 0x7c, 0xfb, 0x41, 0x46, 0x1a, - 0x23, 0x83, 0x89, 0xd7, 0x8c, 0x7e, 0x11, 0x0a, 0x0b, 0xe6, 0xd0, 0xce, 0x33, 0x78, 0xf4, 0xad, - 0x60, 0xa1, 0x06, 0x94, 0x75, 0xb0, 0xe2, 0x2c, 0x17, 0xf0, 0x7a, 0xdf, 0x79, 0x08, 0xb5, 0xad, - 0xc0, 0xa8, 0xb2, 0x91, 0x64, 0x0b, 0xf5, 0xa0, 0x62, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b, - 0x24, 0x33, 0xb6, 0x83, 0x04, 0x24, 0x59, 0x2f, 0x73, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x0a, 0xa7, - 0x21, 0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xbd, 0xec, 0x1c, 0xc7, 0x20, 0x4c, 0xff, 0x18, 0xb9, - 0x9c, 0xca, 0x38, 0x85, 0x78, 0x43, 0x45, 0x2f, 0xa0, 0xc4, 0x69, 0x28, 0x08, 0x17, 0xdf, 0x95, - 0x64, 0x1c, 0x43, 0x26, 0xcc, 0x73, 0xed, 0x4b, 0x9c, 0x30, 0xd0, 0x0b, 0xa8, 0x04, 0x1e, 0xb1, - 0x95, 0x57, 0x6b, 0x57, 0xd1, 0x7f, 0x90, 0x45, 0x9f, 0x24, 0x20, 0xbc, 0xc1, 0xa3, 0xcf, 0x01, - 0x3c, 0x36, 0x9b, 0x3a, 0xdc, 0x5d, 0x52, 0xae, 0x45, 0xd2, 0xc8, 0x62, 0x0f, 0x15, 0x02, 0x57, - 0x3c, 0x36, 0x8b, 0x97, 0xe8, 0xe0, 0xff, 0x52, 0x48, 0x4a, 0x1d, 0x87, 0x00, 0x64, 0xfd, 0x54, - 0xeb, 0xe3, 0x93, 0xef, 0xe5, 0x4a, 0x67, 0x24, 0x45, 0x47, 0x4f, 0xa0, 0x7a, 0xce, 0xb8, 0x4d, - 0xa7, 0x5a, 0xf7, 0x15, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0x2b, 0x50, 0xe2, 0x91, 0x2f, - 0xdc, 0x05, 0xed, 0x1c, 0xc2, 0xbb, 0x99, 0x4e, 0x51, 0x17, 0xaa, 0xeb, 0x34, 0x4f, 0x5d, 0x47, - 0xe9, 0xa3, 0xd2, 0x7f, 0xb8, 0xba, 0x6e, 0x99, 0x6b, 0x3d, 0x8c, 0x87, 0xd8, 0x5c, 0x83, 0xc6, - 0x4e, 0xe7, 0xef, 0x25, 0xa8, 0x6d, 0x89, 0x05, 0xbd, 0x03, 0xbb, 0xee, 0x82, 0xcc, 0x68, 0x4c, - 0xc7, 0xf1, 0x06, 0x8d, 0xa0, 0xe8, 0x91, 0x33, 0xea, 0x49, 0xc9, 0xc8, 0xb0, 0xfd, 0xe4, 0x4e, - 0xd5, 0xed, 0xff, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x97, 0x58, 0x93, 0x91, 0x05, 0x25, 0x9b, 0x2d, - 0x16, 0xc4, 0x97, 0xe5, 0x25, 0xbf, 0x57, 0xc1, 0xc9, 0x16, 0x21, 0x28, 0x10, 0x3e, 0x0b, 0xad, - 0x82, 0x32, 0xab, 0x35, 0xaa, 0x43, 0x9e, 0xfa, 0x4b, 0x6b, 0x57, 0x99, 0xe4, 0x52, 0x5a, 0x1c, - 0x37, 0xce, 0x79, 0x05, 0xcb, 0xa5, 0xe4, 0x45, 0x21, 0xe5, 0x56, 0x49, 0x99, 0xd4, 0x1a, 0xfd, - 0x12, 0x8a, 0x0b, 0x16, 0xf9, 0x22, 0xb4, 0xca, 0xea, 0xb0, 0x8f, 0xb3, 0x0e, 0x7b, 0x24, 0x11, - 0xba, 0xfc, 0x69, 0x38, 0x7a, 0x09, 0x8f, 0x42, 0xc1, 0x82, 0xe9, 0x8c, 0x13, 0x9b, 0x4e, 0x03, - 0xca, 0x5d, 0xe6, 0xa8, 0x6c, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0x87, 0x92, 0x76, 0x20, - 0x59, 0x13, 0x45, 0x42, 0x13, 0xa8, 0x06, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e, - 0xbe, 0x47, 0xd4, 0x26, 0x91, 0xe7, 0x7d, 0x19, 0x93, 0xb0, 0x19, 0x6c, 0x36, 0xe8, 0x3d, 0x28, - 0xce, 0x38, 0x8b, 0x82, 0xd0, 0x32, 0x55, 0x3c, 0xf4, 0x0e, 0x7d, 0x01, 0xa5, 0x90, 0xda, 0x9c, - 0x8a, 0xd0, 0xaa, 0xaa, 0xdb, 0x7e, 0x94, 0xf5, 0x92, 0x13, 0x05, 0xc1, 0xf4, 0x9c, 0x72, 0xea, - 0xdb, 0x14, 0x27, 0x1c, 0xf4, 0x18, 0xf2, 0x42, 0x5c, 0x5a, 0xb5, 0xb6, 0xb1, 0x57, 0xee, 0x97, - 0x56, 0xd7, 0xad, 0xfc, 0xe9, 0xe9, 0x6b, 0x2c, 0x6d, 0xb2, 0x4c, 0xcd, 0x59, 0x28, 0x7c, 0xb2, - 0xa0, 0xd6, 0x03, 0x15, 0xde, 0xf5, 0x1e, 0xbd, 0x06, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2, - 0x7a, 0xa8, 0x6e, 0xf7, 0xe9, 0xdd, 0xb7, 0x1b, 0x1e, 0x9f, 0xe8, 0x62, 0x5b, 0x5b, 0x5d, 0xb7, - 0x2a, 0xeb, 0x2d, 0xae, 0x38, 0x7e, 0x18, 0x2f, 0x51, 0x1f, 0xcc, 0x39, 0x25, 0x9e, 0x98, 0xdb, - 0x73, 0x6a, 0x5f, 0x58, 0xf5, 0xdb, 0x6b, 0xef, 0x4b, 0x05, 0xd3, 0x1e, 0xd2, 0x24, 0x29, 0x62, - 0x79, 0xd4, 0xd0, 0x7a, 0xa4, 0x62, 0x15, 0x6f, 0x1a, 0x9f, 0x83, 0x99, 0x12, 0xa5, 0x14, 0xd3, - 0x05, 0xbd, 0xd4, 0x3a, 0x97, 0x4b, 0x49, 0x5b, 0x12, 0x2f, 0x8a, 0xa7, 0xa9, 0x0a, 0x8e, 0x37, - 0xbf, 0xda, 0x79, 0x6e, 0x34, 0xba, 0x60, 0xa6, 0x32, 0x83, 0x3e, 0x82, 0x1a, 0xa7, 0x33, 0x37, - 0x14, 0xfc, 0x72, 0x4a, 0x22, 0x31, 0xb7, 0x7e, 0xab, 0x08, 0xd5, 0xc4, 0xd8, 0x8b, 0xc4, 0xbc, - 0x31, 0x85, 0xcd, 0x05, 0x51, 0x1b, 0x4c, 0x19, 0xb8, 0x90, 0xf2, 0x25, 0xe5, 0xb2, 0xec, 0xcb, - 0x73, 0xa5, 0x4d, 0x32, 0xc1, 0x21, 0x25, 0xdc, 0x9e, 0xab, 0x4f, 0xac, 0x82, 0xf5, 0x4e, 0x7e, - 0x33, 0x89, 0x8a, 0xf4, 0x37, 0xa3, 0xb7, 0x9d, 0xff, 0x18, 0x50, 0x4d, 0xf7, 0x1f, 0x34, 0x88, - 0xbb, 0x8e, 0xba, 0xd2, 0x83, 0xee, 0xb3, 0xbb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x1d, - 0xc9, 0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x37, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb3, 0x2e, - 0x33, 0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x83, 0x6d, 0x6f, 0xe8, 0x29, 0xe4, 0x5f, 0x8d, - 0x27, 0xf5, 0x5c, 0xe3, 0x83, 0x37, 0x57, 0xed, 0xf7, 0xb7, 0x1f, 0xbe, 0x72, 0xb9, 0x88, 0x88, - 0x37, 0x9e, 0xa0, 0x1f, 0xc3, 0xee, 0xf0, 0xf8, 0x04, 0xe3, 0xba, 0xd1, 0x68, 0xbd, 0xb9, 0x6a, - 0x7f, 0xb0, 0x8d, 0x93, 0x8f, 0x58, 0xe4, 0x3b, 0x98, 0x9d, 0xad, 0xc7, 0xa6, 0xbf, 0xed, 0x80, - 0xa9, 0x0b, 0xe0, 0xfd, 0x8e, 0x4d, 0xbf, 0x81, 0x5a, 0xdc, 0x53, 0x12, 0x59, 0xef, 0xdc, 0xd9, - 0x5a, 0xaa, 0x31, 0x41, 0xe7, 0xf8, 0x09, 0x54, 0xdd, 0x60, 0xf9, 0xd9, 0x94, 0xfa, 0xe4, 0xcc, - 0xd3, 0x13, 0x54, 0x19, 0x9b, 0xd2, 0x36, 0x8a, 0x4d, 0xf2, 0x9b, 0x72, 0x7d, 0x41, 0xb9, 0xaf, - 0x67, 0xa3, 0x32, 0x5e, 0xef, 0xd1, 0x17, 0x50, 0x70, 0x03, 0xb2, 0xd0, 0xfd, 0x30, 0xf3, 0x06, - 0xe3, 0x49, 0xef, 0x48, 0x6b, 0xb0, 0x5f, 0x5e, 0x5d, 0xb7, 0x0a, 0xd2, 0x80, 0x15, 0x0d, 0x35, - 0x93, 0x96, 0x24, 0xdf, 0xa4, 0x4a, 0x64, 0x19, 0xa7, 0x2c, 0x9d, 0xbf, 0x16, 0xc0, 0x1c, 0x78, - 0x51, 0x28, 0x74, 0xa1, 0xbf, 0xb7, 0xb8, 0xbd, 0x86, 0x47, 0x44, 0x0d, 0xd9, 0xc4, 0x97, 0x55, - 0x53, 0xb5, 0x7a, 0x1d, 0xbb, 0xa7, 0x99, 0xee, 0xd6, 0xe0, 0x78, 0x2c, 0xe8, 0x17, 0xa5, 0x4f, - 0xcb, 0xc0, 0x75, 0xf2, 0x3f, 0x4f, 0xd0, 0x09, 0xd4, 0x18, 0xb7, 0xe7, 0x34, 0x14, 0x71, 0xa1, - 0xd5, 0x43, 0x69, 0xe6, 0xef, 0xca, 0x97, 0x69, 0xa0, 0xae, 0x32, 0xf1, 0x69, 0xb7, 0x7d, 0xa0, - 0xe7, 0x50, 0xe0, 0xe4, 0x3c, 0x19, 0x5b, 0x32, 0xf5, 0x8d, 0xc9, 0xb9, 0xd8, 0x72, 0xa1, 0x18, - 0xe8, 0xf7, 0x00, 0x8e, 0x1b, 0x06, 0x44, 0xd8, 0x73, 0xca, 0x75, 0x9e, 0x32, 0xaf, 0x38, 0x5c, - 0xa3, 0xb6, 0xbc, 0xa4, 0xd8, 0xe8, 0x10, 0x2a, 0x36, 0x49, 0x94, 0x56, 0xbc, 0xbd, 0xc7, 0x0c, - 0x7a, 0xda, 0x45, 0x5d, 0xba, 0x58, 0x5d, 0xb7, 0xca, 0x89, 0x05, 0x97, 0x6d, 0xa2, 0x95, 0x77, - 0x08, 0x35, 0x39, 0xc1, 0x4f, 0x1d, 0x7a, 0x4e, 0x22, 0x4f, 0x84, 0xaa, 0x1d, 0xde, 0x52, 0x35, - 0xe5, 0x30, 0x39, 0xd4, 0x38, 0x7d, 0xae, 0xaa, 0x48, 0xd9, 0x3a, 0x2e, 0x40, 0xdc, 0x2e, 0xee, - 0x57, 0x26, 0x08, 0x0a, 0x0e, 0x11, 0x44, 0x29, 0xa3, 0x8a, 0xd5, 0xba, 0xff, 0xe1, 0xdb, 0x9b, - 0x66, 0xee, 0x9f, 0x37, 0xcd, 0xdc, 0xbf, 0x6f, 0x9a, 0xc6, 0x9f, 0x56, 0x4d, 0xe3, 0xed, 0xaa, - 0x69, 0xfc, 0x63, 0xd5, 0x34, 0xfe, 0xb5, 0x6a, 0x1a, 0x67, 0x45, 0xf5, 0xe3, 0xfc, 0xf3, 0xff, - 0x06, 0x00, 0x00, 0xff, 0xff, 0x04, 0xd4, 0x09, 0xa4, 0x97, 0x0f, 0x00, 0x00, + // 1620 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0xdb, 0xc8, + 0x11, 0x26, 0x24, 0x8a, 0x3f, 0x0d, 0xca, 0xa6, 0xa6, 0xf6, 0x07, 0xe6, 0x6e, 0x48, 0x9a, 0xeb, + 0x6c, 0xb4, 0xd9, 0x8a, 0x9c, 0x30, 0xa9, 0x8d, 0x37, 0xce, 0x56, 0xc2, 0xbf, 0xc8, 0x8c, 0x22, + 0x2d, 0x6b, 0xa4, 0x75, 0xca, 0x27, 0xd6, 0x08, 0x18, 0x91, 0x28, 0x81, 0x18, 0x64, 0x30, 0xe0, + 0x16, 0x6f, 0x39, 0x6e, 0xf9, 0x90, 0x37, 0xf0, 0x29, 0xcf, 0x90, 0x4b, 0x9e, 0xc0, 0xc7, 0x1c, + 0x73, 0x52, 0xc5, 0x7c, 0x82, 0x54, 0xe5, 0x01, 0x92, 0x9a, 0xc1, 0x00, 0x04, 0x77, 0xa1, 0xb5, + 0xab, 0xe2, 0xdb, 0x4c, 0xe3, 0xfb, 0x1a, 0x3d, 0x3d, 0x1f, 0xba, 0x1b, 0x60, 0x86, 0x01, 0xb5, + 0xc3, 0xa3, 0x80, 0x33, 0xc1, 0x10, 0x72, 0x98, 0x7d, 0x4d, 0xf9, 0x51, 0xf8, 0x35, 0xe1, 0x8b, + 0x6b, 0x57, 0x1c, 0x2d, 0x7f, 0xd6, 0x30, 0xc5, 0x2a, 0xa0, 0x1a, 0xd0, 0x78, 0x67, 0xc6, 0x66, + 0x4c, 0x2d, 0x1f, 0xca, 0x95, 0xb6, 0xbe, 0xef, 0x44, 0x9c, 0x08, 0x97, 0xf9, 0x0f, 0x93, 0x45, + 0xfc, 0xa0, 0xf3, 0x97, 0x22, 0x54, 0xce, 0x98, 0x43, 0xcf, 0x03, 0x6a, 0xa3, 0x63, 0x30, 0x89, + 0xef, 0x33, 0xa1, 0x00, 0xa1, 0x65, 0xb4, 0x8d, 0x43, 0xb3, 0xdb, 0x3a, 0xfa, 0xee, 0x2b, 0x8f, + 0x7a, 0x1b, 0x58, 0xbf, 0xf8, 0xf2, 0xa6, 0x55, 0xc0, 0x59, 0x26, 0xfa, 0x29, 0x14, 0x39, 0xf3, + 0xa8, 0xb5, 0xd3, 0x36, 0x0e, 0xef, 0x74, 0x3f, 0xcc, 0xf3, 0x20, 0x5f, 0x8a, 0x99, 0x47, 0xb1, + 0x42, 0xa2, 0x63, 0x80, 0x05, 0x5d, 0x5c, 0x52, 0x1e, 0xce, 0xdd, 0xc0, 0xda, 0x55, 0xbc, 0x1f, + 0xdd, 0xc6, 0x93, 0xc1, 0x1e, 0x9d, 0xa6, 0x70, 0x9c, 0xa1, 0xa2, 0x53, 0xa8, 0x91, 0x25, 0x71, + 0x3d, 0x72, 0xe9, 0x7a, 0xae, 0x58, 0x59, 0x45, 0xe5, 0xea, 0x93, 0xef, 0x75, 0xd5, 0xcb, 0x10, + 0xf0, 0x16, 0xbd, 0xe3, 0x00, 0x6c, 0x5e, 0x84, 0x3e, 0x86, 0xf2, 0x64, 0x74, 0x36, 0x1c, 0x9f, + 0x1d, 0xd7, 0x0b, 0x8d, 0x7b, 0xcf, 0x5f, 0xb4, 0xdf, 0x95, 0x3e, 0x36, 0x80, 0x09, 0xf5, 0x1d, + 0xd7, 0x9f, 0xa1, 0x43, 0xa8, 0xf4, 0x06, 0x83, 0xd1, 0xe4, 0x62, 0x34, 0xac, 0x1b, 0x8d, 0xc6, + 0xf3, 0x17, 0xed, 0xf7, 0xb6, 0x81, 0x3d, 0xdb, 0xa6, 0x81, 0xa0, 0x4e, 0xa3, 0xf8, 0xcd, 0x5f, + 0x9b, 0x85, 0xce, 0x37, 0x06, 0xd4, 0xb2, 0x41, 0xa0, 0x8f, 0xa1, 0xd4, 0x1b, 0x5c, 0x8c, 0x9f, + 0x8e, 0xea, 0x85, 0x0d, 0x3d, 0x8b, 0xe8, 0xd9, 0xc2, 0x5d, 0x52, 0xf4, 0x00, 0xf6, 0x26, 0xbd, + 0xaf, 0xce, 0x47, 0x75, 0x63, 0x13, 0x4e, 0x16, 0x36, 0x21, 0x51, 0xa8, 0x50, 0x43, 0xdc, 0x1b, + 0x9f, 0xd5, 0x77, 0xf2, 0x51, 0x43, 0x4e, 0x5c, 0x5f, 0x87, 0xf2, 0x6a, 0x17, 0xcc, 0x73, 0xca, + 0x97, 0xae, 0xfd, 0x96, 0x35, 0xf1, 0x19, 0x14, 0x05, 0x09, 0xaf, 0x95, 0x26, 0xcc, 0x7c, 0x4d, + 0x5c, 0x90, 0xf0, 0x5a, 0xbe, 0x54, 0xd3, 0x15, 0x5e, 0x2a, 0x83, 0xd3, 0xc0, 0x73, 0x6d, 0x22, + 0xa8, 0xa3, 0x94, 0x61, 0x76, 0x7f, 0x98, 0xc7, 0xc6, 0x29, 0x4a, 0xc7, 0xff, 0xa4, 0x80, 0x33, + 0x54, 0xf4, 0x18, 0x4a, 0x33, 0x8f, 0x5d, 0x12, 0x4f, 0x69, 0xc2, 0xec, 0xde, 0xcf, 0x73, 0x72, + 0xac, 0x10, 0x1b, 0x07, 0x9a, 0x82, 0x1e, 0x41, 0x29, 0x0a, 0x1c, 0x22, 0xa8, 0x55, 0x52, 0xe4, + 0x76, 0x1e, 0xf9, 0x2b, 0x85, 0x18, 0x30, 0xff, 0xca, 0x9d, 0x61, 0x8d, 0x47, 0x27, 0x50, 0xf1, + 0xa9, 0xf8, 0x9a, 0xf1, 0xeb, 0xd0, 0x2a, 0xb7, 0x77, 0x0f, 0xcd, 0xee, 0xa7, 0xb9, 0x62, 0x8c, + 0x31, 0x3d, 0x21, 0x88, 0x3d, 0x5f, 0x50, 0x5f, 0xc4, 0x6e, 0xfa, 0x3b, 0x96, 0x81, 0x53, 0x07, + 0xe8, 0xd7, 0x50, 0xa1, 0xbe, 0x13, 0x30, 0xd7, 0x17, 0x56, 0xe5, 0xf6, 0x40, 0x46, 0x1a, 0x23, + 0x93, 0x89, 0x53, 0x46, 0xbf, 0x04, 0xc5, 0x05, 0x73, 0x68, 0xe7, 0x21, 0x1c, 0x7c, 0x27, 0x59, + 0xa8, 0x01, 0x15, 0x9d, 0xac, 0xf8, 0x96, 0x8b, 0x38, 0xdd, 0x77, 0xee, 0xc2, 0xfe, 0x56, 0x62, + 0x54, 0xd9, 0x48, 0x6e, 0x0b, 0xf5, 0xa0, 0x6a, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b, 0x24, + 0x37, 0xb7, 0x83, 0x04, 0x24, 0x59, 0x4f, 0x0a, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x2a, 0xa7, 0x21, + 0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xc3, 0xfc, 0x3b, 0x8e, 0x41, 0x98, 0xfe, 0x29, 0x72, 0x39, + 0x95, 0x79, 0x0a, 0xf1, 0x86, 0x8a, 0x1e, 0x43, 0x99, 0xd3, 0x50, 0x10, 0x2e, 0xbe, 0xef, 0x92, + 0x71, 0x0c, 0x99, 0x30, 0xcf, 0xb5, 0x57, 0x38, 0x61, 0xa0, 0xc7, 0x50, 0x0d, 0x3c, 0x62, 0x2b, + 0xaf, 0xd6, 0x9e, 0xa2, 0xff, 0x20, 0x8f, 0x3e, 0x49, 0x40, 0x78, 0x83, 0x47, 0x9f, 0x03, 0x78, + 0x6c, 0x36, 0x75, 0xb8, 0xbb, 0xa4, 0x5c, 0x8b, 0xa4, 0x91, 0xc7, 0x1e, 0x2a, 0x04, 0xae, 0x7a, + 0x6c, 0x16, 0x2f, 0xd1, 0xf1, 0xff, 0xa5, 0x90, 0x8c, 0x3a, 0x4e, 0x00, 0x48, 0xfa, 0x54, 0xeb, + 0xe3, 0x93, 0x37, 0x72, 0xa5, 0x6f, 0x24, 0x43, 0x47, 0xf7, 0xa1, 0x76, 0xc5, 0xb8, 0x4d, 0xa7, + 0x5a, 0xf7, 0x55, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0xab, 0x50, 0xe6, 0x91, 0x2f, 0xdc, + 0x05, 0xed, 0x9c, 0xc0, 0xbb, 0xb9, 0x4e, 0x51, 0x17, 0x6a, 0xe9, 0x35, 0x4f, 0x5d, 0x47, 0xe9, + 0xa3, 0xda, 0xbf, 0xbb, 0xbe, 0x69, 0x99, 0xa9, 0x1e, 0xc6, 0x43, 0x6c, 0xa6, 0xa0, 0xb1, 0xd3, + 0xf9, 0x7b, 0x19, 0xf6, 0xb7, 0xc4, 0x82, 0xde, 0x81, 0x3d, 0x77, 0x41, 0x66, 0x34, 0xa6, 0xe3, + 0x78, 0x83, 0x46, 0x50, 0xf2, 0xc8, 0x25, 0xf5, 0xa4, 0x64, 0x64, 0xda, 0x7e, 0xf2, 0x5a, 0xd5, + 0x1d, 0xfd, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x2b, 0xac, 0xc9, 0xc8, 0x82, 0xb2, 0xcd, 0x16, 0x0b, + 0xe2, 0xcb, 0xf2, 0xb2, 0x7b, 0x58, 0xc5, 0xc9, 0x16, 0x21, 0x28, 0x12, 0x3e, 0x0b, 0xad, 0xa2, + 0x32, 0xab, 0x35, 0xaa, 0xc3, 0x2e, 0xf5, 0x97, 0xd6, 0x9e, 0x32, 0xc9, 0xa5, 0xb4, 0x38, 0x6e, + 0x7c, 0xe7, 0x55, 0x2c, 0x97, 0x92, 0x17, 0x85, 0x94, 0x5b, 0x65, 0x65, 0x52, 0x6b, 0xf4, 0x4b, + 0x28, 0x2d, 0x58, 0xe4, 0x8b, 0xd0, 0xaa, 0xa8, 0x60, 0xef, 0xe5, 0x05, 0x7b, 0x2a, 0x11, 0xba, + 0xfc, 0x69, 0x38, 0x7a, 0x02, 0x07, 0xa1, 0x60, 0xc1, 0x74, 0xc6, 0x89, 0x4d, 0xa7, 0x01, 0xe5, + 0x2e, 0x73, 0xd4, 0x6d, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0xbb, 0x92, 0x76, 0x2c, 0x59, + 0x13, 0x45, 0x42, 0x13, 0xa8, 0x05, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e, 0xde, + 0x20, 0x6b, 0x93, 0xc8, 0xf3, 0xbe, 0x8c, 0x49, 0xd8, 0x0c, 0x36, 0x1b, 0xf4, 0x1e, 0x94, 0x66, + 0x9c, 0x45, 0x41, 0x68, 0x99, 0x2a, 0x1f, 0x7a, 0x87, 0xbe, 0x80, 0x72, 0x48, 0x6d, 0x4e, 0x45, + 0x68, 0xd5, 0xd4, 0x69, 0x3f, 0xca, 0x7b, 0xc9, 0xb9, 0x82, 0x60, 0x7a, 0x45, 0x39, 0xf5, 0x6d, + 0x8a, 0x13, 0x0e, 0xba, 0x07, 0xbb, 0x42, 0xac, 0xac, 0xfd, 0xb6, 0x71, 0x58, 0xe9, 0x97, 0xd7, + 0x37, 0xad, 0xdd, 0x8b, 0x8b, 0x67, 0x58, 0xda, 0x64, 0x99, 0x9a, 0xb3, 0x50, 0xf8, 0x64, 0x41, + 0xad, 0x3b, 0x2a, 0xbd, 0xe9, 0x1e, 0x3d, 0x03, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2, 0xba, + 0xab, 0x4e, 0xf7, 0xe9, 0xeb, 0x4f, 0x37, 0x3c, 0x3b, 0xd7, 0xc5, 0x76, 0x7f, 0x7d, 0xd3, 0xaa, + 0xa6, 0x5b, 0x5c, 0x75, 0xfc, 0x30, 0x5e, 0xa2, 0x3e, 0x98, 0x73, 0x4a, 0x3c, 0x31, 0xb7, 0xe7, + 0xd4, 0xbe, 0xb6, 0xea, 0xb7, 0xd7, 0xde, 0x27, 0x0a, 0xa6, 0x3d, 0x64, 0x49, 0x52, 0xc4, 0x32, + 0xd4, 0xd0, 0x3a, 0x50, 0xb9, 0x8a, 0x37, 0x8d, 0xcf, 0xc1, 0xcc, 0x88, 0x52, 0x8a, 0xe9, 0x9a, + 0xae, 0xb4, 0xce, 0xe5, 0x52, 0xd2, 0x96, 0xc4, 0x8b, 0xe2, 0x69, 0xaa, 0x8a, 0xe3, 0xcd, 0xaf, + 0x76, 0x1e, 0x19, 0x8d, 0x2e, 0x98, 0x99, 0x9b, 0x41, 0x1f, 0xc1, 0x3e, 0xa7, 0x33, 0x37, 0x14, + 0x7c, 0x35, 0x25, 0x91, 0x98, 0x5b, 0xbf, 0x55, 0x84, 0x5a, 0x62, 0xec, 0x45, 0x62, 0xde, 0x98, + 0xc2, 0xe6, 0x80, 0xa8, 0x0d, 0xa6, 0x4c, 0x5c, 0x48, 0xf9, 0x92, 0x72, 0x59, 0xf6, 0x65, 0x5c, + 0x59, 0x93, 0xbc, 0xe0, 0x90, 0x12, 0x6e, 0xcf, 0xd5, 0x27, 0x56, 0xc5, 0x7a, 0x27, 0xbf, 0x99, + 0x44, 0x45, 0xfa, 0x9b, 0xd1, 0xdb, 0xce, 0x7f, 0x0c, 0xa8, 0x65, 0xfb, 0x0f, 0x1a, 0xc4, 0x5d, + 0x47, 0x1d, 0xe9, 0x4e, 0xf7, 0xe1, 0xeb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x9d, 0xca, + 0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x2f, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb7, 0x2e, 0x33, + 0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x9d, 0x6d, 0x6f, 0xe8, 0x01, 0xec, 0x3e, 0x1d, 0x4f, + 0xea, 0x85, 0xc6, 0x07, 0xcf, 0x5f, 0xb4, 0xdf, 0xdf, 0x7e, 0xf8, 0xd4, 0xe5, 0x22, 0x22, 0xde, + 0x78, 0x82, 0x7e, 0x0c, 0x7b, 0xc3, 0xb3, 0x73, 0x8c, 0xeb, 0x46, 0xa3, 0xf5, 0xfc, 0x45, 0xfb, + 0x83, 0x6d, 0x9c, 0x7c, 0xc4, 0x22, 0xdf, 0xc1, 0xec, 0x32, 0x1d, 0x9b, 0xfe, 0xb6, 0x03, 0xa6, + 0x2e, 0x80, 0x6f, 0x77, 0x6c, 0xfa, 0x0d, 0xec, 0xc7, 0x3d, 0x25, 0x91, 0xf5, 0xce, 0x6b, 0x5b, + 0x4b, 0x2d, 0x26, 0xe8, 0x3b, 0xbe, 0x0f, 0x35, 0x37, 0x58, 0x7e, 0x36, 0xa5, 0x3e, 0xb9, 0xf4, + 0xf4, 0x04, 0x55, 0xc1, 0xa6, 0xb4, 0x8d, 0x62, 0x93, 0xfc, 0xa6, 0x5c, 0x5f, 0x50, 0xee, 0xeb, + 0xd9, 0xa8, 0x82, 0xd3, 0x3d, 0xfa, 0x02, 0x8a, 0x6e, 0x40, 0x16, 0xba, 0x1f, 0xe6, 0x9e, 0x60, + 0x3c, 0xe9, 0x9d, 0x6a, 0x0d, 0xf6, 0x2b, 0xeb, 0x9b, 0x56, 0x51, 0x1a, 0xb0, 0xa2, 0xa1, 0x66, + 0xd2, 0x92, 0xe4, 0x9b, 0x54, 0x89, 0xac, 0xe0, 0x8c, 0xa5, 0xf3, 0xdf, 0x22, 0x98, 0x03, 0x2f, + 0x0a, 0x85, 0x2e, 0xf4, 0x6f, 0x2d, 0x6f, 0xcf, 0xe0, 0x80, 0xa8, 0x21, 0x9b, 0xf8, 0xb2, 0x6a, + 0xaa, 0x56, 0xaf, 0x73, 0xf7, 0x20, 0xd7, 0x5d, 0x0a, 0x8e, 0xc7, 0x82, 0x7e, 0x49, 0xfa, 0xb4, + 0x0c, 0x5c, 0x27, 0xdf, 0x7a, 0x82, 0xce, 0x61, 0x9f, 0x71, 0x7b, 0x4e, 0x43, 0x11, 0x17, 0x5a, + 0x3d, 0x94, 0xe6, 0xfe, 0xae, 0x7c, 0x99, 0x05, 0xea, 0x2a, 0x13, 0x47, 0xbb, 0xed, 0x03, 0x3d, + 0x82, 0x22, 0x27, 0x57, 0xc9, 0xd8, 0x92, 0xab, 0x6f, 0x4c, 0xae, 0xc4, 0x96, 0x0b, 0xc5, 0x40, + 0xbf, 0x07, 0x70, 0xdc, 0x30, 0x20, 0xc2, 0x9e, 0x53, 0xae, 0xef, 0x29, 0xf7, 0x88, 0xc3, 0x14, + 0xb5, 0xe5, 0x25, 0xc3, 0x46, 0x27, 0x50, 0xb5, 0x49, 0xa2, 0xb4, 0xd2, 0xed, 0x3d, 0x66, 0xd0, + 0xd3, 0x2e, 0xea, 0xd2, 0xc5, 0xfa, 0xa6, 0x55, 0x49, 0x2c, 0xb8, 0x62, 0x13, 0xad, 0xbc, 0x13, + 0xd8, 0x97, 0x13, 0xfc, 0xd4, 0xa1, 0x57, 0x24, 0xf2, 0x44, 0xa8, 0xda, 0xe1, 0x2d, 0x55, 0x53, + 0x0e, 0x93, 0x43, 0x8d, 0xd3, 0x71, 0xd5, 0x44, 0xc6, 0x86, 0xfe, 0x08, 0x07, 0xd4, 0xb7, 0xf9, + 0x4a, 0xe9, 0x2c, 0x89, 0xb0, 0x72, 0xfb, 0x61, 0x47, 0x29, 0x78, 0xeb, 0xb0, 0x75, 0xfa, 0x2d, + 0x7b, 0xc7, 0x05, 0x88, 0xfb, 0xd0, 0xdb, 0xd5, 0x1f, 0x82, 0xa2, 0x43, 0x04, 0x51, 0x92, 0xab, + 0x61, 0xb5, 0xee, 0x7f, 0xf8, 0xf2, 0x55, 0xb3, 0xf0, 0xcf, 0x57, 0xcd, 0xc2, 0xbf, 0x5f, 0x35, + 0x8d, 0x3f, 0xaf, 0x9b, 0xc6, 0xcb, 0x75, 0xd3, 0xf8, 0xc7, 0xba, 0x69, 0xfc, 0x6b, 0xdd, 0x34, + 0x2e, 0x4b, 0xea, 0x8f, 0xfc, 0xe7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xce, 0x13, 0x97, 0xa2, + 0xf0, 0x0f, 0x00, 0x00, } diff --git a/api/specs.proto b/api/specs.proto index 4e3d0dded4..ae2b7874a3 100644 --- a/api/specs.proto +++ b/api/specs.proto @@ -323,6 +323,9 @@ message ClusterSpec { // TaskDefaults specifies the default values to use for task creation. TaskDefaults task_defaults = 7 [(gogoproto.nullable) = false]; + + // EncryptionConfig defines the cluster's encryption settings. + EncryptionConfig encryption_config = 8 [(gogoproto.nullable) = false]; } // SecretSpec specifies a user-provided secret. diff --git a/api/types.pb.go b/api/types.pb.go index 7fbe601325..f1df343734 100644 --- a/api/types.pb.go +++ b/api/types.pb.go @@ -52,6 +52,7 @@ TaskDefaults DispatcherConfig RaftConfig + EncryptionConfig Placement JoinTokens RootCA @@ -118,7 +119,7 @@ GetClusterResponse ListClustersRequest ListClustersResponse - JoinTokenRotation + KeyRotation UpdateClusterRequest UpdateClusterResponse GetSecretRequest @@ -149,6 +150,8 @@ IssueNodeCertificateResponse GetRootCACertificateRequest GetRootCACertificateResponse + GetUnlockKeyRequest + GetUnlockKeyResponse StoreSnapshot ClusterSnapshot Snapshot @@ -662,7 +665,7 @@ func (x EncryptionKey_Algorithm) String() string { return proto.EnumName(EncryptionKey_Algorithm_name, int32(x)) } func (EncryptionKey_Algorithm) EnumDescriptor() ([]byte, []int) { - return fileDescriptorTypes, []int{37, 0} + return fileDescriptorTypes, []int{38, 0} } type MaybeEncryptedRecord_Algorithm int32 @@ -685,7 +688,7 @@ func (x MaybeEncryptedRecord_Algorithm) String() string { return proto.EnumName(MaybeEncryptedRecord_Algorithm_name, int32(x)) } func (MaybeEncryptedRecord_Algorithm) EnumDescriptor() ([]byte, []int) { - return fileDescriptorTypes, []int{42, 0} + return fileDescriptorTypes, []int{43, 0} } // Version tracks the last time an object in the store was updated. @@ -1359,6 +1362,17 @@ func (m *RaftConfig) Reset() { *m = RaftConfig{} } func (*RaftConfig) ProtoMessage() {} func (*RaftConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{32} } +type EncryptionConfig struct { + // AutoLockManagers specifies whether or not managers TLS keys and raft data + // should be encrypted at rest in such a way that they must be unlocked + // before the manager node starts up again. + AutoLockManagers bool `protobuf:"varint,1,opt,name=auto_lock_managers,json=autoLockManagers,proto3" json:"auto_lock_managers,omitempty"` +} + +func (m *EncryptionConfig) Reset() { *m = EncryptionConfig{} } +func (*EncryptionConfig) ProtoMessage() {} +func (*EncryptionConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} } + // Placement specifies task distribution constraints. type Placement struct { // constraints specifies a set of requirements a node should meet for a task. @@ -1367,7 +1381,7 @@ type Placement struct { func (m *Placement) Reset() { *m = Placement{} } func (*Placement) ProtoMessage() {} -func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} } +func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} } // JoinToken contains the join tokens for workers and managers. type JoinTokens struct { @@ -1379,7 +1393,7 @@ type JoinTokens struct { func (m *JoinTokens) Reset() { *m = JoinTokens{} } func (*JoinTokens) ProtoMessage() {} -func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} } +func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} } type RootCA struct { // CAKey is the root CA private key. @@ -1394,7 +1408,7 @@ type RootCA struct { func (m *RootCA) Reset() { *m = RootCA{} } func (*RootCA) ProtoMessage() {} -func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} } +func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} } type Certificate struct { Role NodeRole `protobuf:"varint,1,opt,name=role,proto3,enum=docker.swarmkit.v1.NodeRole" json:"role,omitempty"` @@ -1407,7 +1421,7 @@ type Certificate struct { func (m *Certificate) Reset() { *m = Certificate{} } func (*Certificate) ProtoMessage() {} -func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} } +func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} } // Symmetric keys to encrypt inter-agent communication. type EncryptionKey struct { @@ -1423,7 +1437,7 @@ type EncryptionKey struct { func (m *EncryptionKey) Reset() { *m = EncryptionKey{} } func (*EncryptionKey) ProtoMessage() {} -func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} } +func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} } // ManagerStatus provides informations about the state of a manager in the cluster. type ManagerStatus struct { @@ -1440,7 +1454,7 @@ type ManagerStatus struct { func (m *ManagerStatus) Reset() { *m = ManagerStatus{} } func (*ManagerStatus) ProtoMessage() {} -func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} } +func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} } // SecretReference is the linkage between a service and a secret that it uses. type SecretReference struct { @@ -1460,7 +1474,7 @@ type SecretReference struct { func (m *SecretReference) Reset() { *m = SecretReference{} } func (*SecretReference) ProtoMessage() {} -func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} } +func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} } type isSecretReference_Target interface { isSecretReference_Target() @@ -1558,7 +1572,7 @@ type SecretReference_FileTarget struct { func (m *SecretReference_FileTarget) Reset() { *m = SecretReference_FileTarget{} } func (*SecretReference_FileTarget) ProtoMessage() {} func (*SecretReference_FileTarget) Descriptor() ([]byte, []int) { - return fileDescriptorTypes, []int{39, 0} + return fileDescriptorTypes, []int{40, 0} } // BlacklistedCertificate is a record for a blacklisted certificate. It does not @@ -1571,7 +1585,7 @@ type BlacklistedCertificate struct { func (m *BlacklistedCertificate) Reset() { *m = BlacklistedCertificate{} } func (*BlacklistedCertificate) ProtoMessage() {} -func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} } +func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} } // HealthConfig holds configuration settings for the HEALTHCHECK feature. type HealthConfig struct { @@ -1595,7 +1609,7 @@ type HealthConfig struct { func (m *HealthConfig) Reset() { *m = HealthConfig{} } func (*HealthConfig) ProtoMessage() {} -func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} } +func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} } type MaybeEncryptedRecord struct { Algorithm MaybeEncryptedRecord_Algorithm `protobuf:"varint,1,opt,name=algorithm,proto3,enum=docker.swarmkit.v1.MaybeEncryptedRecord_Algorithm" json:"algorithm,omitempty"` @@ -1605,7 +1619,7 @@ type MaybeEncryptedRecord struct { func (m *MaybeEncryptedRecord) Reset() { *m = MaybeEncryptedRecord{} } func (*MaybeEncryptedRecord) ProtoMessage() {} -func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} } +func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{43} } func init() { proto.RegisterType((*Version)(nil), "docker.swarmkit.v1.Version") @@ -1646,6 +1660,7 @@ func init() { proto.RegisterType((*TaskDefaults)(nil), "docker.swarmkit.v1.TaskDefaults") proto.RegisterType((*DispatcherConfig)(nil), "docker.swarmkit.v1.DispatcherConfig") proto.RegisterType((*RaftConfig)(nil), "docker.swarmkit.v1.RaftConfig") + proto.RegisterType((*EncryptionConfig)(nil), "docker.swarmkit.v1.EncryptionConfig") proto.RegisterType((*Placement)(nil), "docker.swarmkit.v1.Placement") proto.RegisterType((*JoinTokens)(nil), "docker.swarmkit.v1.JoinTokens") proto.RegisterType((*RootCA)(nil), "docker.swarmkit.v1.RootCA") @@ -2276,6 +2291,18 @@ func (m *RaftConfig) Copy() *RaftConfig { return o } +func (m *EncryptionConfig) Copy() *EncryptionConfig { + if m == nil { + return nil + } + + o := &EncryptionConfig{ + AutoLockManagers: m.AutoLockManagers, + } + + return o +} + func (m *Placement) Copy() *Placement { if m == nil { return nil @@ -3028,6 +3055,16 @@ func (this *RaftConfig) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *EncryptionConfig) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&api.EncryptionConfig{") + s = append(s, "AutoLockManagers: "+fmt.Sprintf("%#v", this.AutoLockManagers)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func (this *Placement) GoString() string { if this == nil { return "nil" @@ -4708,6 +4745,34 @@ func (m *RaftConfig) MarshalTo(data []byte) (int, error) { return i, nil } +func (m *EncryptionConfig) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *EncryptionConfig) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.AutoLockManagers { + data[i] = 0x8 + i++ + if m.AutoLockManagers { + data[i] = 1 + } else { + data[i] = 0 + } + i++ + } + return i, nil +} + func (m *Placement) Marshal() (data []byte, err error) { size := m.Size() data = make([]byte, size) @@ -5836,6 +5901,15 @@ func (m *RaftConfig) Size() (n int) { return n } +func (m *EncryptionConfig) Size() (n int) { + var l int + _ = l + if m.AutoLockManagers { + n += 2 + } + return n +} + func (m *Placement) Size() (n int) { var l int _ = l @@ -6569,6 +6643,16 @@ func (this *RaftConfig) String() string { }, "") return s } +func (this *EncryptionConfig) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&EncryptionConfig{`, + `AutoLockManagers:` + fmt.Sprintf("%v", this.AutoLockManagers) + `,`, + `}`, + }, "") + return s +} func (this *Placement) String() string { if this == nil { return "nil" @@ -11863,6 +11947,76 @@ func (m *RaftConfig) Unmarshal(data []byte) error { } return nil } +func (m *EncryptionConfig) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: EncryptionConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: EncryptionConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AutoLockManagers", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.AutoLockManagers = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipTypes(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Placement) Unmarshal(data []byte) error { l := len(data) iNdEx := 0 @@ -13478,252 +13632,254 @@ var ( func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 3946 bytes of a gzipped FileDescriptorProto + // 3975 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x79, 0x4d, 0x6c, 0x1b, 0x49, - 0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0xcf, - 0x78, 0x67, 0xc6, 0x3b, 0x7f, 0x8e, 0xad, 0xf9, 0x80, 0x77, 0xfc, 0xcf, 0x7a, 0x9a, 0x1f, 0xb2, - 0xb8, 0x96, 0x48, 0xa2, 0x48, 0xd9, 0x19, 0x04, 0x08, 0x51, 0xea, 0x2e, 0x91, 0x3d, 0x6a, 0x76, - 0x33, 0xdd, 0x45, 0xc9, 0x4c, 0x10, 0xc4, 0xc8, 0x21, 0x09, 0x74, 0xca, 0x3d, 0x10, 0x82, 0x20, - 0x41, 0x0e, 0x39, 0xec, 0x25, 0x87, 0x00, 0x39, 0x0d, 0x72, 0x9a, 0xe3, 0x26, 0x01, 0x82, 0x45, - 0x82, 0x18, 0x19, 0xe5, 0x1c, 0x60, 0x2f, 0x8b, 0x1c, 0x92, 0x00, 0x41, 0x7d, 0x74, 0xb3, 0x29, - 0xd3, 0xb2, 0x27, 0xbb, 0x17, 0xb2, 0xeb, 0xd5, 0xef, 0xbd, 0xfa, 0x7a, 0x55, 0xf5, 0x7b, 0xaf, - 0xa0, 0xc0, 0xa6, 0x63, 0x1a, 0x54, 0xc6, 0xbe, 0xc7, 0x3c, 0x84, 0x2c, 0xcf, 0x3c, 0xa2, 0x7e, - 0x25, 0x38, 0x21, 0xfe, 0xe8, 0xc8, 0x66, 0x95, 0xe3, 0xbb, 0xa5, 0x1b, 0xcc, 0x1e, 0xd1, 0x80, - 0x91, 0xd1, 0xf8, 0xa3, 0xe8, 0x4b, 0xc2, 0x4b, 0x6f, 0x58, 0x13, 0x9f, 0x30, 0xdb, 0x73, 0x3f, - 0x0a, 0x3f, 0x54, 0xc5, 0xb5, 0x81, 0x37, 0xf0, 0xc4, 0xe7, 0x47, 0xfc, 0x4b, 0x4a, 0xf5, 0x4d, - 0x58, 0x7e, 0x4c, 0xfd, 0xc0, 0xf6, 0x5c, 0x74, 0x0d, 0x32, 0xb6, 0x6b, 0xd1, 0xa7, 0xeb, 0x89, - 0x72, 0xe2, 0xfd, 0x34, 0x96, 0x05, 0xfd, 0xcf, 0x12, 0x50, 0x30, 0x5c, 0xd7, 0x63, 0xc2, 0x56, - 0x80, 0x10, 0xa4, 0x5d, 0x32, 0xa2, 0x02, 0x94, 0xc7, 0xe2, 0x1b, 0xd5, 0x20, 0xeb, 0x90, 0x03, - 0xea, 0x04, 0xeb, 0xc9, 0x72, 0xea, 0xfd, 0xc2, 0xd6, 0x0f, 0x2a, 0x2f, 0xf6, 0xb9, 0x12, 0x33, - 0x52, 0xd9, 0x15, 0xe8, 0x86, 0xcb, 0xfc, 0x29, 0x56, 0xaa, 0xa5, 0x1f, 0x42, 0x21, 0x26, 0x46, - 0x1a, 0xa4, 0x8e, 0xe8, 0x54, 0x35, 0xc3, 0x3f, 0x79, 0xff, 0x8e, 0x89, 0x33, 0xa1, 0xeb, 0x49, - 0x21, 0x93, 0x85, 0xcf, 0x93, 0xf7, 0x12, 0xfa, 0x97, 0x90, 0xc7, 0x34, 0xf0, 0x26, 0xbe, 0x49, - 0x03, 0xf4, 0x01, 0xe4, 0x5d, 0xe2, 0x7a, 0x7d, 0x73, 0x3c, 0x09, 0x84, 0x7a, 0xaa, 0x5a, 0x3c, - 0x7f, 0xbe, 0x99, 0x6b, 0x11, 0xd7, 0xab, 0x75, 0xf6, 0x03, 0x9c, 0xe3, 0xd5, 0xb5, 0xf1, 0x24, - 0x40, 0x6f, 0x43, 0x71, 0x44, 0x47, 0x9e, 0x3f, 0xed, 0x1f, 0x4c, 0x19, 0x0d, 0x84, 0xe1, 0x14, - 0x2e, 0x48, 0x59, 0x95, 0x8b, 0xf4, 0x3f, 0x4e, 0xc0, 0xb5, 0xd0, 0x36, 0xa6, 0xbf, 0x35, 0xb1, - 0x7d, 0x3a, 0xa2, 0x2e, 0x0b, 0xd0, 0xa7, 0x90, 0x75, 0xec, 0x91, 0xcd, 0x64, 0x1b, 0x85, 0xad, - 0xb7, 0x16, 0x8d, 0x39, 0xea, 0x15, 0x56, 0x60, 0x64, 0x40, 0xd1, 0xa7, 0x01, 0xf5, 0x8f, 0xe5, - 0x4c, 0x88, 0x26, 0x5f, 0xa9, 0x3c, 0xa7, 0xa2, 0x6f, 0x43, 0xae, 0xe3, 0x10, 0x76, 0xe8, 0xf9, - 0x23, 0xa4, 0x43, 0x91, 0xf8, 0xe6, 0xd0, 0x66, 0xd4, 0x64, 0x13, 0x3f, 0x5c, 0x95, 0x39, 0x19, - 0xba, 0x0e, 0x49, 0x4f, 0x36, 0x94, 0xaf, 0x66, 0xcf, 0x9f, 0x6f, 0x26, 0xdb, 0x5d, 0x9c, 0xf4, - 0x02, 0xfd, 0x3e, 0x5c, 0xe9, 0x38, 0x93, 0x81, 0xed, 0xd6, 0x69, 0x60, 0xfa, 0xf6, 0x98, 0x5b, - 0xe7, 0xcb, 0xcb, 0x9d, 0x2f, 0x5c, 0x5e, 0xfe, 0x1d, 0x2d, 0x79, 0x72, 0xb6, 0xe4, 0xfa, 0x1f, - 0x26, 0xe1, 0x4a, 0xc3, 0x1d, 0xd8, 0x2e, 0x8d, 0x6b, 0xdf, 0x82, 0x55, 0x2a, 0x84, 0xfd, 0x63, - 0xe9, 0x54, 0xca, 0xce, 0x8a, 0x94, 0x86, 0x9e, 0xd6, 0xbc, 0xe0, 0x2f, 0x77, 0x17, 0x0d, 0xff, - 0x05, 0xeb, 0x8b, 0xbc, 0x06, 0x35, 0x60, 0x79, 0x2c, 0x06, 0x11, 0xac, 0xa7, 0x84, 0xad, 0x5b, - 0x8b, 0x6c, 0xbd, 0x30, 0xce, 0x6a, 0xfa, 0x9b, 0xe7, 0x9b, 0x4b, 0x38, 0xd4, 0xfd, 0x65, 0x9c, - 0xef, 0xdf, 0x13, 0xb0, 0xd6, 0xf2, 0xac, 0xb9, 0x79, 0x28, 0x41, 0x6e, 0xe8, 0x05, 0x2c, 0xb6, - 0x51, 0xa2, 0x32, 0xba, 0x07, 0xb9, 0xb1, 0x5a, 0x3e, 0xb5, 0xfa, 0x37, 0x17, 0x77, 0x59, 0x62, - 0x70, 0x84, 0x46, 0xf7, 0x21, 0xef, 0x87, 0x3e, 0xb1, 0x9e, 0x7a, 0x1d, 0xc7, 0x99, 0xe1, 0xd1, - 0xaf, 0x41, 0x56, 0x2e, 0xc2, 0x7a, 0x5a, 0x68, 0xde, 0x7a, 0xad, 0x39, 0xc7, 0x4a, 0x49, 0xff, - 0x59, 0x02, 0x34, 0x4c, 0x0e, 0xd9, 0x1e, 0x1d, 0x1d, 0x50, 0xbf, 0xcb, 0x08, 0x9b, 0x04, 0xe8, - 0x3a, 0x64, 0x1d, 0x4a, 0x2c, 0xea, 0x8b, 0x41, 0xe6, 0xb0, 0x2a, 0xa1, 0x7d, 0xee, 0xe4, 0xc4, - 0x1c, 0x92, 0x03, 0xdb, 0xb1, 0xd9, 0x54, 0x0c, 0x73, 0x75, 0xf1, 0x2a, 0x5f, 0xb4, 0x59, 0xc1, - 0x31, 0x45, 0x3c, 0x67, 0x06, 0xad, 0xc3, 0xf2, 0x88, 0x06, 0x01, 0x19, 0x50, 0x31, 0xfa, 0x3c, - 0x0e, 0x8b, 0xfa, 0x7d, 0x28, 0xc6, 0xf5, 0x50, 0x01, 0x96, 0xf7, 0x5b, 0x8f, 0x5a, 0xed, 0x27, - 0x2d, 0x6d, 0x09, 0xad, 0x41, 0x61, 0xbf, 0x85, 0x1b, 0x46, 0x6d, 0xc7, 0xa8, 0xee, 0x36, 0xb4, - 0x04, 0x5a, 0x81, 0xfc, 0xac, 0x98, 0xd4, 0xff, 0x3a, 0x01, 0xc0, 0x17, 0x50, 0x0d, 0xea, 0x73, - 0xc8, 0x04, 0x8c, 0x30, 0xb9, 0x70, 0xab, 0x5b, 0xef, 0x2e, 0xea, 0xf5, 0x0c, 0x5e, 0xe1, 0x7f, - 0x14, 0x4b, 0x95, 0x78, 0x0f, 0x93, 0x73, 0x3d, 0xe4, 0x7b, 0x88, 0x58, 0x96, 0xaf, 0x3a, 0x2e, - 0xbe, 0xf5, 0xfb, 0x90, 0x11, 0xda, 0xf3, 0xdd, 0xcd, 0x41, 0xba, 0xce, 0xbf, 0x12, 0x28, 0x0f, - 0x19, 0xdc, 0x30, 0xea, 0x5f, 0x6a, 0x49, 0xa4, 0x41, 0xb1, 0xde, 0xec, 0xd6, 0xda, 0xad, 0x56, - 0xa3, 0xd6, 0x6b, 0xd4, 0xb5, 0x94, 0x7e, 0x0b, 0x32, 0xcd, 0x11, 0xb7, 0x7c, 0x93, 0x7b, 0xc5, - 0x21, 0xf5, 0xa9, 0x6b, 0x86, 0xce, 0x36, 0x13, 0xe8, 0x3f, 0xcd, 0x43, 0x66, 0xcf, 0x9b, 0xb8, - 0x0c, 0x6d, 0xc5, 0x76, 0xf6, 0xea, 0xd6, 0xc6, 0xa2, 0x61, 0x09, 0x60, 0xa5, 0x37, 0x1d, 0x53, - 0xb5, 0xf3, 0xaf, 0x43, 0x56, 0xfa, 0x8f, 0x1a, 0x8e, 0x2a, 0x71, 0x39, 0x23, 0xfe, 0x80, 0x32, - 0x35, 0x1e, 0x55, 0x42, 0xef, 0x43, 0xce, 0xa7, 0xc4, 0xf2, 0x5c, 0x67, 0x2a, 0xdc, 0x2c, 0x27, - 0x8f, 0x5e, 0x4c, 0x89, 0xd5, 0x76, 0x9d, 0x29, 0x8e, 0x6a, 0xd1, 0x0e, 0x14, 0x0f, 0x6c, 0xd7, - 0xea, 0x7b, 0x63, 0x79, 0x0e, 0x66, 0x5e, 0xee, 0x94, 0xb2, 0x57, 0x55, 0xdb, 0xb5, 0xda, 0x12, - 0x8c, 0x0b, 0x07, 0xb3, 0x02, 0x6a, 0xc1, 0xea, 0xb1, 0xe7, 0x4c, 0x46, 0x34, 0xb2, 0x95, 0x15, - 0xb6, 0xde, 0x7b, 0xb9, 0xad, 0xc7, 0x02, 0x1f, 0x5a, 0x5b, 0x39, 0x8e, 0x17, 0xd1, 0x23, 0x58, - 0x61, 0xa3, 0xf1, 0x61, 0x10, 0x99, 0x5b, 0x16, 0xe6, 0xbe, 0x7f, 0xc9, 0x84, 0x71, 0x78, 0x68, - 0xad, 0xc8, 0x62, 0xa5, 0xd2, 0xef, 0xa7, 0xa0, 0x10, 0xeb, 0x39, 0xea, 0x42, 0x61, 0xec, 0x7b, - 0x63, 0x32, 0x10, 0x67, 0xb9, 0x5a, 0x8b, 0xbb, 0xaf, 0x35, 0xea, 0x4a, 0x67, 0xa6, 0x88, 0xe3, - 0x56, 0xf4, 0xb3, 0x24, 0x14, 0x62, 0x95, 0xe8, 0x36, 0xe4, 0x70, 0x07, 0x37, 0x1f, 0x1b, 0xbd, - 0x86, 0xb6, 0x54, 0xba, 0x79, 0x7a, 0x56, 0x5e, 0x17, 0xd6, 0xe2, 0x06, 0x3a, 0xbe, 0x7d, 0xcc, - 0x5d, 0xef, 0x7d, 0x58, 0x0e, 0xa1, 0x89, 0xd2, 0x9b, 0xa7, 0x67, 0xe5, 0x37, 0x2e, 0x42, 0x63, - 0x48, 0xdc, 0xdd, 0x31, 0x70, 0xa3, 0xae, 0x25, 0x17, 0x23, 0x71, 0x77, 0x48, 0x7c, 0x6a, 0xa1, - 0xef, 0x43, 0x56, 0x01, 0x53, 0xa5, 0xd2, 0xe9, 0x59, 0xf9, 0xfa, 0x45, 0xe0, 0x0c, 0x87, 0xbb, - 0xbb, 0xc6, 0xe3, 0x86, 0x96, 0x5e, 0x8c, 0xc3, 0x5d, 0x87, 0x1c, 0x53, 0xf4, 0x2e, 0x64, 0x24, - 0x2c, 0x53, 0xba, 0x71, 0x7a, 0x56, 0xfe, 0xde, 0x0b, 0xe6, 0x38, 0xaa, 0xb4, 0xfe, 0x47, 0x7f, - 0xbe, 0xb1, 0xf4, 0xb7, 0x7f, 0xb1, 0xa1, 0x5d, 0xac, 0x2e, 0xfd, 0x77, 0x02, 0x56, 0xe6, 0x96, - 0x1c, 0xe9, 0x90, 0x75, 0x3d, 0xd3, 0x1b, 0xcb, 0x23, 0x3e, 0x57, 0x85, 0xf3, 0xe7, 0x9b, 0xd9, - 0x96, 0x57, 0xf3, 0xc6, 0x53, 0xac, 0x6a, 0xd0, 0xa3, 0x0b, 0x97, 0xd4, 0xc7, 0xaf, 0xe9, 0x4f, - 0x0b, 0xaf, 0xa9, 0x07, 0xb0, 0x62, 0xf9, 0xf6, 0x31, 0xf5, 0xfb, 0xa6, 0xe7, 0x1e, 0xda, 0x03, - 0x75, 0x7c, 0x97, 0x16, 0xd9, 0xac, 0x0b, 0x20, 0x2e, 0x4a, 0x85, 0x9a, 0xc0, 0xff, 0x12, 0x17, - 0x54, 0xe9, 0x31, 0x14, 0xe3, 0x1e, 0x8a, 0xde, 0x02, 0x08, 0xec, 0xdf, 0xa6, 0x8a, 0xf3, 0x08, - 0x86, 0x84, 0xf3, 0x5c, 0x22, 0x18, 0x0f, 0x7a, 0x0f, 0xd2, 0x23, 0xcf, 0x92, 0x76, 0x56, 0xaa, - 0x57, 0xf9, 0x3d, 0xf9, 0xcf, 0xcf, 0x37, 0x0b, 0x5e, 0x50, 0xd9, 0xb6, 0x1d, 0xba, 0xe7, 0x59, - 0x14, 0x0b, 0x80, 0x7e, 0x0c, 0x69, 0x7e, 0x54, 0xa0, 0x37, 0x21, 0x5d, 0x6d, 0xb6, 0xea, 0xda, - 0x52, 0xe9, 0xca, 0xe9, 0x59, 0x79, 0x45, 0x4c, 0x09, 0xaf, 0xe0, 0xbe, 0x8b, 0x36, 0x21, 0xfb, - 0xb8, 0xbd, 0xbb, 0xbf, 0xc7, 0xdd, 0xeb, 0xea, 0xe9, 0x59, 0x79, 0x2d, 0xaa, 0x96, 0x93, 0x86, - 0xde, 0x82, 0x4c, 0x6f, 0xaf, 0xb3, 0xdd, 0xd5, 0x92, 0x25, 0x74, 0x7a, 0x56, 0x5e, 0x8d, 0xea, - 0x45, 0x9f, 0x4b, 0x57, 0xd4, 0xaa, 0xe6, 0x23, 0xb9, 0xfe, 0x5f, 0x49, 0x58, 0xc1, 0x9c, 0xf3, - 0xfa, 0xac, 0xe3, 0x39, 0xb6, 0x39, 0x45, 0x1d, 0xc8, 0x9b, 0x9e, 0x6b, 0xd9, 0xb1, 0x3d, 0xb5, - 0xf5, 0x92, 0x8b, 0x71, 0xa6, 0x15, 0x96, 0x6a, 0xa1, 0x26, 0x9e, 0x19, 0x41, 0x5b, 0x90, 0xb1, - 0xa8, 0x43, 0xa6, 0x97, 0xdd, 0xd0, 0x75, 0xc5, 0xaf, 0xb1, 0x84, 0x0a, 0x36, 0x49, 0x9e, 0xf6, - 0x09, 0x63, 0x74, 0x34, 0x66, 0xf2, 0x86, 0x4e, 0xe3, 0xc2, 0x88, 0x3c, 0x35, 0x94, 0x08, 0x7d, - 0x02, 0xd9, 0x13, 0xdb, 0xb5, 0xbc, 0x13, 0x75, 0x09, 0x5f, 0x6e, 0x57, 0x61, 0xf5, 0x53, 0x7e, - 0xf7, 0x5e, 0xe8, 0x2c, 0x9f, 0xf5, 0x56, 0xbb, 0xd5, 0x08, 0x67, 0x5d, 0xd5, 0xb7, 0xdd, 0x96, - 0xe7, 0xf2, 0x1d, 0x03, 0xed, 0x56, 0x7f, 0xdb, 0x68, 0xee, 0xee, 0x63, 0x3e, 0xf3, 0xd7, 0x4e, - 0xcf, 0xca, 0x5a, 0x04, 0xd9, 0x26, 0xb6, 0xc3, 0x89, 0xe1, 0x0d, 0x48, 0x19, 0xad, 0x2f, 0xb5, - 0x64, 0x49, 0x3b, 0x3d, 0x2b, 0x17, 0xa3, 0x6a, 0xc3, 0x9d, 0xce, 0x36, 0xd3, 0xc5, 0x76, 0xf5, - 0x7f, 0x4d, 0x42, 0x71, 0x7f, 0x6c, 0x11, 0x46, 0xa5, 0x67, 0xa2, 0x32, 0x14, 0xc6, 0xc4, 0x27, - 0x8e, 0x43, 0x1d, 0x3b, 0x18, 0xa9, 0xe0, 0x21, 0x2e, 0x42, 0xf7, 0xbe, 0xc3, 0x64, 0x2a, 0x62, - 0xa6, 0xa6, 0x74, 0x1f, 0x56, 0x0f, 0x65, 0x67, 0xfb, 0xc4, 0x14, 0xab, 0x9b, 0x12, 0xab, 0x5b, - 0x59, 0x64, 0x22, 0xde, 0xab, 0x8a, 0x1a, 0xa3, 0x21, 0xb4, 0xf0, 0xca, 0x61, 0xbc, 0x88, 0x3e, - 0x83, 0xe5, 0x91, 0xe7, 0xda, 0xcc, 0xf3, 0x5f, 0x6b, 0x1d, 0x42, 0x30, 0xba, 0x0d, 0x57, 0xf8, - 0x0a, 0x87, 0x5d, 0x12, 0xd5, 0xe2, 0xe6, 0x4a, 0xe2, 0xb5, 0x11, 0x79, 0xaa, 0xda, 0xc4, 0x5c, - 0xac, 0x7f, 0x06, 0x2b, 0x73, 0x7d, 0xe0, 0xb7, 0x79, 0xc7, 0xd8, 0xef, 0x36, 0xb4, 0x25, 0x54, - 0x84, 0x5c, 0xad, 0xdd, 0xea, 0x35, 0x5b, 0xfb, 0x9c, 0x8e, 0x14, 0x21, 0x87, 0xdb, 0xbb, 0xbb, - 0x55, 0xa3, 0xf6, 0x48, 0x4b, 0xea, 0xbf, 0x88, 0xe6, 0x57, 0xf1, 0x91, 0xea, 0x3c, 0x1f, 0xf9, - 0xf0, 0xe5, 0x43, 0x57, 0x8c, 0x64, 0x56, 0x88, 0x78, 0xc9, 0xff, 0x07, 0x10, 0xcb, 0x48, 0xad, - 0x3e, 0x61, 0x97, 0xc5, 0x1c, 0xbd, 0x30, 0x9a, 0xc4, 0x79, 0xa5, 0x60, 0x30, 0xf4, 0x05, 0x14, - 0x4d, 0x6f, 0x34, 0x76, 0xa8, 0xd2, 0x4f, 0xbd, 0x8e, 0x7e, 0x21, 0x52, 0x31, 0x58, 0x9c, 0x17, - 0xa5, 0xe7, 0x99, 0xdb, 0x1f, 0x24, 0xa0, 0x10, 0xeb, 0xf0, 0x3c, 0x15, 0x2a, 0x42, 0x6e, 0xbf, - 0x53, 0x37, 0x7a, 0xcd, 0xd6, 0x43, 0x2d, 0x81, 0x00, 0xb2, 0x62, 0x02, 0xeb, 0x5a, 0x92, 0x53, - 0xb8, 0x5a, 0x7b, 0xaf, 0xb3, 0xdb, 0x10, 0x64, 0x08, 0x5d, 0x03, 0x2d, 0x9c, 0xc2, 0x7e, 0xb7, - 0x67, 0x60, 0x2e, 0x4d, 0xa3, 0xab, 0xb0, 0x16, 0x49, 0x95, 0x66, 0x06, 0x5d, 0x07, 0x14, 0x09, - 0x67, 0x26, 0xb2, 0xfa, 0xef, 0xc2, 0x5a, 0xcd, 0x73, 0x19, 0xb1, 0xdd, 0x88, 0xde, 0x6e, 0xf1, - 0x71, 0x2b, 0x51, 0xdf, 0xb6, 0xe4, 0x69, 0x5b, 0x5d, 0x3b, 0x7f, 0xbe, 0x59, 0x88, 0xa0, 0xcd, - 0x3a, 0x1f, 0x69, 0x58, 0xb0, 0xf8, 0x9e, 0x1a, 0xdb, 0x96, 0x98, 0xe2, 0x4c, 0x75, 0xf9, 0xfc, - 0xf9, 0x66, 0xaa, 0xd3, 0xac, 0x63, 0x2e, 0x43, 0x6f, 0x42, 0x9e, 0x3e, 0xb5, 0x59, 0xdf, 0xe4, - 0xa7, 0x2b, 0x9f, 0xc3, 0x0c, 0xce, 0x71, 0x41, 0x8d, 0x1f, 0xa6, 0x55, 0x80, 0x8e, 0xe7, 0x33, - 0xd5, 0xf2, 0x27, 0x90, 0x19, 0x7b, 0xbe, 0x88, 0x2d, 0xf9, 0xd5, 0xb3, 0x90, 0xac, 0x71, 0xb8, - 0x74, 0x76, 0x2c, 0xc1, 0xfa, 0xdf, 0x25, 0x01, 0x7a, 0x24, 0x38, 0x52, 0x46, 0xee, 0x43, 0x3e, - 0x4a, 0x0e, 0x5c, 0x16, 0xa4, 0xc6, 0xd6, 0x3c, 0xc2, 0xa3, 0x8f, 0x43, 0xaf, 0x93, 0xdc, 0x7d, - 0xb1, 0xa2, 0x6a, 0x6b, 0x11, 0xfd, 0x9d, 0x27, 0xe8, 0xfc, 0xbe, 0xa2, 0xbe, 0xaf, 0x16, 0x9f, - 0x7f, 0xa2, 0x9a, 0x38, 0xb3, 0xe5, 0xbc, 0x29, 0xf6, 0xf7, 0xce, 0xa2, 0x46, 0x2e, 0x2c, 0xca, - 0xce, 0x12, 0x9e, 0xe9, 0xa1, 0x07, 0x50, 0xe0, 0x43, 0xef, 0x07, 0xa2, 0x4e, 0x11, 0xbf, 0x97, - 0xce, 0x96, 0xb4, 0x80, 0x61, 0x1c, 0x7d, 0x57, 0x35, 0x58, 0xf5, 0x27, 0x2e, 0x1f, 0xb6, 0xb2, - 0xa1, 0xdb, 0xf0, 0x46, 0x8b, 0xb2, 0x13, 0xcf, 0x3f, 0x32, 0x18, 0x23, 0xe6, 0x90, 0x47, 0xfb, - 0xea, 0xa4, 0x9b, 0xb1, 0xde, 0xc4, 0x1c, 0xeb, 0x5d, 0x87, 0x65, 0xe2, 0xd8, 0x24, 0xa0, 0x92, - 0x2a, 0xe4, 0x71, 0x58, 0xe4, 0xdc, 0x9c, 0x33, 0x7d, 0x1a, 0x04, 0x54, 0xc6, 0xa7, 0x79, 0x3c, - 0x13, 0xe8, 0xff, 0x98, 0x04, 0x68, 0x76, 0x8c, 0x3d, 0x65, 0xbe, 0x0e, 0xd9, 0x43, 0x32, 0xb2, - 0x9d, 0xe9, 0x65, 0x3b, 0x7d, 0x86, 0xaf, 0x18, 0xd2, 0xd0, 0xb6, 0xd0, 0xc1, 0x4a, 0x57, 0x50, - 0xf6, 0xc9, 0x81, 0x4b, 0x59, 0x44, 0xd9, 0x45, 0x89, 0xf3, 0x03, 0x9f, 0xb8, 0xd1, 0xca, 0xc8, - 0x02, 0xef, 0xfa, 0x80, 0x30, 0x7a, 0x42, 0xa6, 0xe1, 0xc6, 0x54, 0x45, 0xb4, 0xc3, 0xa9, 0x7c, - 0x40, 0xfd, 0x63, 0x6a, 0xad, 0x67, 0x84, 0x17, 0xbe, 0xaa, 0x3f, 0x58, 0xc1, 0x25, 0xf3, 0x89, - 0xb4, 0x4b, 0xf7, 0xc5, 0x75, 0x3d, 0xab, 0xfa, 0x4e, 0xd1, 0xf5, 0x1d, 0x58, 0x99, 0x1b, 0xe7, - 0x0b, 0xb1, 0x52, 0xb3, 0xf3, 0xf8, 0x13, 0x2d, 0xad, 0xbe, 0x3e, 0xd3, 0xb2, 0xfa, 0x5f, 0xa5, - 0xe4, 0x56, 0x52, 0xb3, 0xba, 0x38, 0x5f, 0x95, 0x13, 0xd9, 0x2f, 0xd3, 0x73, 0x94, 0x7f, 0xbf, - 0x77, 0xf9, 0x0e, 0xe3, 0xdc, 0x5b, 0xc0, 0x71, 0xa4, 0x88, 0x36, 0xa1, 0x20, 0xd7, 0xbf, 0xcf, - 0xfd, 0x49, 0x4c, 0xeb, 0x0a, 0x06, 0x29, 0xe2, 0x9a, 0xe8, 0x16, 0xac, 0x8e, 0x27, 0x07, 0x8e, - 0x1d, 0x0c, 0xa9, 0x25, 0x31, 0x69, 0x81, 0x59, 0x89, 0xa4, 0x02, 0xb6, 0x07, 0x45, 0x25, 0xe8, - 0x0b, 0xde, 0x95, 0x11, 0x1d, 0xba, 0xfd, 0xaa, 0x0e, 0x49, 0x15, 0x41, 0xc7, 0x0a, 0xe3, 0x59, - 0x41, 0xaf, 0x43, 0x2e, 0xec, 0x2c, 0x5a, 0x87, 0x54, 0xaf, 0xd6, 0xd1, 0x96, 0x4a, 0x6b, 0xa7, - 0x67, 0xe5, 0x42, 0x28, 0xee, 0xd5, 0x3a, 0xbc, 0x66, 0xbf, 0xde, 0xd1, 0x12, 0xf3, 0x35, 0xfb, - 0xf5, 0x4e, 0x29, 0xcd, 0x6f, 0x7e, 0xfd, 0x10, 0x0a, 0xb1, 0x16, 0xd0, 0x3b, 0xb0, 0xdc, 0x6c, - 0x3d, 0xc4, 0x8d, 0x6e, 0x57, 0x5b, 0x2a, 0x5d, 0x3f, 0x3d, 0x2b, 0xa3, 0x58, 0x6d, 0xd3, 0x1d, - 0xf0, 0xf5, 0x41, 0x6f, 0x41, 0x7a, 0xa7, 0xdd, 0xed, 0x85, 0x44, 0x2f, 0x86, 0xd8, 0xf1, 0x02, - 0x56, 0xba, 0xaa, 0x28, 0x45, 0xdc, 0xb0, 0xfe, 0x27, 0x09, 0xc8, 0x4a, 0xbe, 0xbb, 0x70, 0xa1, - 0x0c, 0x58, 0x0e, 0xa3, 0x30, 0x49, 0xc2, 0xdf, 0x7b, 0x39, 0x61, 0xae, 0x28, 0x7e, 0x2b, 0xdd, - 0x2f, 0xd4, 0x2b, 0x7d, 0x0e, 0xc5, 0x78, 0xc5, 0x77, 0x72, 0xbe, 0xdf, 0x81, 0x02, 0xf7, 0xef, - 0x90, 0x38, 0x6f, 0x41, 0x56, 0x72, 0x72, 0x75, 0x9a, 0x5e, 0xc6, 0xde, 0x15, 0x12, 0xdd, 0x83, - 0x65, 0xc9, 0xf8, 0xc3, 0xfc, 0xd4, 0xc6, 0xe5, 0xbb, 0x08, 0x87, 0x70, 0xfd, 0x01, 0xa4, 0x3b, - 0x94, 0xfa, 0x7c, 0xee, 0x5d, 0xcf, 0xa2, 0xb3, 0x0b, 0x48, 0x05, 0x2b, 0x16, 0x6d, 0xd6, 0x79, - 0xb0, 0x62, 0xd1, 0xa6, 0x15, 0xa5, 0x17, 0x92, 0xb1, 0xf4, 0x42, 0x0f, 0x8a, 0x4f, 0xa8, 0x3d, - 0x18, 0x32, 0x6a, 0x09, 0x43, 0x1f, 0x42, 0x7a, 0x4c, 0xa3, 0xce, 0xaf, 0x2f, 0x74, 0x30, 0x4a, - 0x7d, 0x2c, 0x50, 0xfc, 0x1c, 0x39, 0x11, 0xda, 0x2a, 0x2b, 0xaa, 0x4a, 0xfa, 0x3f, 0x24, 0x61, - 0xb5, 0x19, 0x04, 0x13, 0xe2, 0x9a, 0x21, 0x43, 0xf9, 0xd1, 0x3c, 0x43, 0x79, 0x7f, 0xe1, 0x08, - 0xe7, 0x54, 0xe6, 0xb3, 0x26, 0xea, 0x72, 0x48, 0x46, 0x97, 0x83, 0xfe, 0x1f, 0x89, 0x30, 0x35, - 0x72, 0x2b, 0xb6, 0xdd, 0x4b, 0xeb, 0xa7, 0x67, 0xe5, 0x6b, 0x71, 0x4b, 0x74, 0xdf, 0x3d, 0x72, - 0xbd, 0x13, 0x17, 0xbd, 0x0d, 0x19, 0xdc, 0x68, 0x35, 0x9e, 0x68, 0x09, 0xe9, 0x9e, 0x73, 0x20, - 0x4c, 0x5d, 0x7a, 0xc2, 0x2d, 0x75, 0x1a, 0xad, 0x3a, 0xe7, 0x12, 0xc9, 0x05, 0x96, 0x3a, 0xd4, - 0xb5, 0x6c, 0x77, 0x80, 0xde, 0x81, 0x6c, 0xb3, 0xdb, 0xdd, 0x17, 0xc1, 0xeb, 0x1b, 0xa7, 0x67, - 0xe5, 0xab, 0x73, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x02, 0x10, 0xe7, - 0x7d, 0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0x16, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb, - 0x97, 0x24, 0x68, 0x86, 0x69, 0xd2, 0x31, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xcc, 0xbf, - 0x6c, 0x1a, 0xf2, 0x80, 0x7b, 0x0b, 0xf3, 0xea, 0x17, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x8d, - 0xec, 0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0xba, 0x00, 0x81, - 0xee, 0x40, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0x6f, 0xbe, 0x2c, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2, - 0x0d, 0x00, 0x32, 0x61, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x81, 0x6c, - 0x40, 0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xc1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9, - 0x52, 0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde, - 0x44, 0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x65, 0xd4, 0x77, - 0x89, 0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0xb0, 0x30, 0x17, 0x1a, 0x69, 0x54, - 0x6a, 0xc6, 0x82, 0xf3, 0xff, 0x06, 0xa4, 0x26, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe3, - 0x5d, 0xcc, 0x65, 0xa8, 0x31, 0x3b, 0xb6, 0x52, 0x2f, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f, - 0xba, 0x3e, 0x04, 0x98, 0xf5, 0x1a, 0x6d, 0x40, 0xa6, 0xb6, 0xdd, 0xed, 0xee, 0x6a, 0x4b, 0xf2, - 0x6c, 0x9e, 0x55, 0x09, 0xb1, 0xfe, 0x97, 0x09, 0xc8, 0xd5, 0x0c, 0x75, 0x63, 0x6e, 0x83, 0x26, - 0x0e, 0x1c, 0x93, 0xfa, 0xac, 0x4f, 0x9f, 0x8e, 0x6d, 0x7f, 0xaa, 0xce, 0x8c, 0xcb, 0xc3, 0xa4, - 0x55, 0xae, 0x55, 0xa3, 0x3e, 0x6b, 0x08, 0x1d, 0x84, 0xa1, 0x48, 0xd5, 0x10, 0xfb, 0x26, 0x09, - 0x4f, 0xf0, 0x8d, 0xcb, 0xa7, 0x42, 0xd2, 0xeb, 0x59, 0x39, 0xc0, 0x85, 0xd0, 0x48, 0x8d, 0x04, - 0xfa, 0x63, 0xb8, 0xda, 0xf6, 0xcd, 0x21, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x01, 0xdc, 0x64, - 0x24, 0x38, 0xea, 0x0f, 0xed, 0x80, 0x79, 0xfe, 0xb4, 0xef, 0x53, 0x46, 0x5d, 0x5e, 0xdf, 0x17, - 0xcf, 0x2e, 0x2a, 0xc9, 0x71, 0x83, 0x63, 0x76, 0x24, 0x04, 0x87, 0x88, 0x5d, 0x0e, 0xd0, 0x9b, - 0x50, 0xe4, 0x6c, 0xb6, 0x4e, 0x0f, 0xc9, 0xc4, 0x61, 0x01, 0xfa, 0x21, 0x80, 0xe3, 0x0d, 0xfa, - 0xaf, 0x7d, 0xdc, 0xe7, 0x1d, 0x6f, 0x20, 0x3f, 0xf5, 0xdf, 0x00, 0xad, 0x6e, 0x07, 0x63, 0xc2, - 0xcc, 0x61, 0x98, 0xbd, 0x41, 0x0f, 0x41, 0x1b, 0x52, 0xe2, 0xb3, 0x03, 0x4a, 0x58, 0x7f, 0x4c, - 0x7d, 0xdb, 0xb3, 0x5e, 0x6b, 0x4a, 0xd7, 0x22, 0xad, 0x8e, 0x50, 0xd2, 0xff, 0x33, 0x01, 0x80, - 0xc9, 0x61, 0x48, 0x6e, 0x7e, 0x00, 0x57, 0x02, 0x97, 0x8c, 0x83, 0xa1, 0xc7, 0xfa, 0xb6, 0xcb, - 0xa8, 0x7f, 0x4c, 0x1c, 0x15, 0x81, 0x6b, 0x61, 0x45, 0x53, 0xc9, 0xd1, 0x87, 0x80, 0x8e, 0x28, - 0x1d, 0xf7, 0x3d, 0xc7, 0xea, 0x87, 0x95, 0xf2, 0x5d, 0x28, 0x8d, 0x35, 0x5e, 0xd3, 0x76, 0xac, - 0x6e, 0x28, 0x47, 0x55, 0xd8, 0xe0, 0x33, 0x40, 0x5d, 0xe6, 0xdb, 0x34, 0xe8, 0x1f, 0x7a, 0x7e, - 0x3f, 0x70, 0xbc, 0x93, 0xfe, 0xa1, 0xe7, 0x38, 0xde, 0x09, 0xf5, 0xc3, 0xfc, 0x46, 0xc9, 0xf1, - 0x06, 0x0d, 0x09, 0xda, 0xf6, 0xfc, 0xae, 0xe3, 0x9d, 0x6c, 0x87, 0x08, 0xce, 0x80, 0x66, 0xc3, - 0x66, 0xb6, 0x79, 0x14, 0x32, 0xa0, 0x48, 0xda, 0xb3, 0xcd, 0x23, 0xf4, 0x0e, 0xac, 0x50, 0x87, - 0x8a, 0x28, 0x59, 0xa2, 0x32, 0x02, 0x55, 0x0c, 0x85, 0x1c, 0xa4, 0xff, 0x3f, 0xc8, 0x77, 0x1c, - 0x62, 0x8a, 0xd7, 0x37, 0x54, 0x06, 0x1e, 0x74, 0x71, 0x27, 0xb0, 0x5d, 0x15, 0x25, 0xe5, 0x71, - 0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x23, 0xea, 0x8a, 0x87, 0x0a, 0xce, - 0xe8, 0xd5, 0x52, 0xe6, 0xb1, 0x2a, 0x89, 0x80, 0x85, 0xb8, 0x64, 0x40, 0xfd, 0x28, 0x5f, 0x2f, - 0x8b, 0xfa, 0x37, 0x09, 0xc8, 0x62, 0xcf, 0x63, 0x35, 0x03, 0x95, 0x21, 0x6b, 0x92, 0x7e, 0xb8, - 0xf3, 0x8a, 0xd5, 0xfc, 0xf9, 0xf3, 0xcd, 0x4c, 0xcd, 0x78, 0x44, 0xa7, 0x38, 0x63, 0x92, 0x47, - 0x74, 0xca, 0xaf, 0x68, 0x93, 0x88, 0xfd, 0x22, 0xcc, 0x14, 0xe5, 0x15, 0x5d, 0x33, 0xf8, 0x66, - 0xc0, 0x59, 0x93, 0xf0, 0x7f, 0x74, 0x07, 0x8a, 0x0a, 0xd4, 0x1f, 0x92, 0x60, 0x28, 0x79, 0x78, - 0x75, 0xf5, 0xfc, 0xf9, 0x26, 0x48, 0xe4, 0x0e, 0x09, 0x86, 0x18, 0x24, 0x9a, 0x7f, 0xa3, 0x06, - 0x14, 0xbe, 0xf2, 0x6c, 0xb7, 0xcf, 0xc4, 0x20, 0x54, 0xaa, 0x62, 0xe1, 0xfe, 0x99, 0x0d, 0x55, - 0xe5, 0x4f, 0xe0, 0xab, 0x48, 0xa2, 0xff, 0x53, 0x02, 0x0a, 0xdc, 0xa6, 0x7d, 0x68, 0x9b, 0xfc, - 0x4a, 0xfd, 0xee, 0x27, 0xfd, 0x0d, 0x48, 0x99, 0x81, 0xaf, 0xc6, 0x26, 0x8e, 0xba, 0x5a, 0x17, - 0x63, 0x2e, 0x43, 0x5f, 0x40, 0x56, 0x05, 0x5f, 0xf2, 0x90, 0xd7, 0x5f, 0x7d, 0xf9, 0xab, 0x2e, - 0x2a, 0x3d, 0xb1, 0x96, 0xb3, 0xde, 0x89, 0x51, 0x16, 0x71, 0x5c, 0x84, 0xae, 0x43, 0xd2, 0x74, - 0x85, 0x53, 0xa8, 0x07, 0xcc, 0x5a, 0x0b, 0x27, 0x4d, 0x57, 0xff, 0xfb, 0x04, 0xac, 0x34, 0x5c, - 0xd3, 0x9f, 0x8a, 0x43, 0x92, 0x2f, 0xc4, 0x4d, 0xc8, 0x07, 0x93, 0x83, 0x60, 0x1a, 0x30, 0x3a, - 0x0a, 0xdf, 0x42, 0x22, 0x01, 0x6a, 0x42, 0x9e, 0x38, 0x03, 0xcf, 0xb7, 0xd9, 0x70, 0xa4, 0x78, - 0xff, 0xe2, 0x83, 0x39, 0x6e, 0xb3, 0x62, 0x84, 0x2a, 0x78, 0xa6, 0x1d, 0x1e, 0xc5, 0x29, 0xd1, - 0x59, 0x71, 0x14, 0xbf, 0x0d, 0x45, 0x87, 0x8c, 0x44, 0x34, 0xca, 0xc3, 0x49, 0x31, 0x8e, 0x34, - 0x2e, 0x28, 0x19, 0x8f, 0xb1, 0x75, 0x1d, 0xf2, 0x91, 0x31, 0xb4, 0x06, 0x05, 0xa3, 0xd1, 0xed, - 0xdf, 0xdd, 0xba, 0xd7, 0x7f, 0x58, 0xdb, 0xd3, 0x96, 0x14, 0x13, 0xf8, 0x9b, 0x04, 0xac, 0xec, - 0x49, 0x1f, 0x54, 0xec, 0xea, 0x1d, 0x58, 0xf6, 0xc9, 0x21, 0x0b, 0xf9, 0x5f, 0x5a, 0x3a, 0x17, - 0x3f, 0x04, 0x38, 0xff, 0xe3, 0x55, 0x8b, 0xf9, 0x5f, 0xec, 0x75, 0x2e, 0x75, 0xe9, 0xeb, 0x5c, - 0xfa, 0x57, 0xf2, 0x3a, 0xa7, 0xff, 0x24, 0x09, 0x6b, 0xea, 0xa2, 0x0e, 0x5f, 0x9f, 0xd0, 0x07, - 0x90, 0x97, 0x77, 0xf6, 0x8c, 0xbd, 0x8a, 0x07, 0x21, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37, - 0x2d, 0x1e, 0x4e, 0x29, 0x68, 0xec, 0xad, 0x19, 0xa4, 0xa8, 0xc5, 0x63, 0x81, 0x3a, 0xa4, 0x0f, - 0x6d, 0x87, 0x2a, 0x3f, 0x5b, 0x98, 0x01, 0xbc, 0xd0, 0xbc, 0x48, 0x58, 0xf7, 0x44, 0x40, 0xb6, - 0xb3, 0x84, 0x85, 0x76, 0xe9, 0xf7, 0x00, 0x66, 0xd2, 0x85, 0x31, 0x07, 0xbf, 0xd7, 0x55, 0x06, - 0x27, 0xbc, 0xd7, 0x9b, 0x75, 0xcc, 0x65, 0xbc, 0x6a, 0x60, 0x5b, 0x6a, 0xe7, 0x8a, 0xaa, 0x87, - 0xbc, 0x6a, 0x60, 0x5b, 0x51, 0xd6, 0x3c, 0xfd, 0x8a, 0xac, 0x79, 0x35, 0x17, 0x26, 0x11, 0xf4, - 0x36, 0x5c, 0xaf, 0x3a, 0xc4, 0x3c, 0x72, 0xec, 0x80, 0x51, 0x2b, 0xbe, 0x43, 0x3f, 0x85, 0xec, - 0xdc, 0xbd, 0xfb, 0x8a, 0xb4, 0x8d, 0x02, 0xeb, 0x3f, 0x49, 0x40, 0x71, 0x87, 0x12, 0x87, 0x0d, - 0x67, 0xb1, 0x2f, 0xa3, 0x01, 0x53, 0xe7, 0xa3, 0xf8, 0x46, 0xf7, 0x20, 0x17, 0xdd, 0x14, 0xaf, - 0x93, 0xdc, 0x8e, 0xd0, 0xe8, 0x33, 0x58, 0xe6, 0x9e, 0xed, 0x4d, 0x42, 0x42, 0xf7, 0x8a, 0xac, - 0xa9, 0x02, 0xf3, 0x43, 0xd6, 0xa7, 0xe2, 0x82, 0x10, 0xb3, 0x93, 0xc1, 0x61, 0x51, 0xff, 0x9f, - 0x04, 0x5c, 0xdb, 0x23, 0xd3, 0x03, 0xaa, 0x76, 0x1c, 0xb5, 0x30, 0x35, 0x3d, 0xdf, 0x42, 0x9d, - 0xf8, 0x4e, 0xbd, 0x24, 0xa1, 0xbf, 0x48, 0x79, 0xf1, 0x86, 0x0d, 0x99, 0x62, 0x32, 0xc6, 0x14, - 0xaf, 0x41, 0xc6, 0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28, - 0xcb, 0x2e, 0x72, 0xe4, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x2f, 0xa0, 0xd4, 0x6d, 0xd4, 0x70, 0xa3, - 0x57, 0x6d, 0xff, 0x7a, 0xbf, 0x6b, 0xec, 0x76, 0x8d, 0xad, 0x3b, 0xfd, 0x4e, 0x7b, 0xf7, 0xcb, - 0xbb, 0x1f, 0xdf, 0xf9, 0x54, 0x4b, 0x94, 0xca, 0xa7, 0x67, 0xe5, 0x9b, 0x2d, 0xa3, 0xb6, 0x2b, - 0xdd, 0xf2, 0xc0, 0x7b, 0xda, 0x25, 0x4e, 0x40, 0xb6, 0xee, 0x74, 0x3c, 0x67, 0xca, 0x31, 0xb7, - 0x7f, 0x91, 0x82, 0x7c, 0x94, 0x44, 0xe3, 0xde, 0xc5, 0x23, 0x18, 0xd5, 0x54, 0x24, 0x6f, 0xd1, - 0x13, 0xf4, 0xf6, 0x2c, 0x76, 0xf9, 0x42, 0x26, 0xf3, 0xa3, 0xea, 0x30, 0x6e, 0x79, 0x17, 0x72, - 0x46, 0xb7, 0xdb, 0x7c, 0xd8, 0x6a, 0xd4, 0xb5, 0xaf, 0x13, 0xa5, 0xef, 0x9d, 0x9e, 0x95, 0xaf, - 0x44, 0x20, 0x23, 0x08, 0xec, 0x81, 0x4b, 0x2d, 0x81, 0xaa, 0xd5, 0x1a, 0x9d, 0x5e, 0xa3, 0xae, - 0x3d, 0x4b, 0x5e, 0x44, 0x09, 0x2e, 0x2e, 0x1e, 0xe6, 0xf2, 0x1d, 0xdc, 0xe8, 0x18, 0x98, 0x37, - 0xf8, 0x75, 0x52, 0x86, 0x54, 0xb3, 0x16, 0x7d, 0x3a, 0x26, 0x3e, 0x6f, 0x73, 0x23, 0x7c, 0xa0, - 0x7e, 0x96, 0x92, 0x8f, 0x37, 0xb3, 0x8c, 0x20, 0x25, 0xd6, 0x94, 0xb7, 0x26, 0xb2, 0xb1, 0xc2, - 0x4c, 0xea, 0x42, 0x6b, 0x5d, 0x46, 0x7c, 0xc6, 0xad, 0xe8, 0xb0, 0x8c, 0xf7, 0x5b, 0x2d, 0x0e, - 0x7a, 0x96, 0xbe, 0x30, 0x3a, 0x3c, 0x71, 0x5d, 0x8e, 0xb9, 0x05, 0xb9, 0x30, 0x59, 0xab, 0x7d, - 0x9d, 0xbe, 0xd0, 0xa1, 0x5a, 0x98, 0x69, 0x16, 0x0d, 0xee, 0xec, 0xf7, 0xc4, 0xfb, 0xf9, 0xb3, - 0xcc, 0xc5, 0x06, 0x87, 0x13, 0x66, 0xf1, 0x60, 0xb1, 0x1c, 0x45, 0x6f, 0x5f, 0x67, 0x24, 0x1f, - 0x8e, 0x30, 0x2a, 0x74, 0x7b, 0x17, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0xd4, 0xfe, 0x2c, 0x7b, 0xc1, - 0x0e, 0xa6, 0x5f, 0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x1d, 0x43, 0x4c, 0xf9, 0x45, 0x54, 0xdb, - 0x1f, 0x0f, 0x89, 0x4b, 0xad, 0xd9, 0x0b, 0x56, 0x54, 0x75, 0xfb, 0x37, 0x21, 0x17, 0xde, 0xb0, - 0x68, 0x03, 0xb2, 0x4f, 0xda, 0xf8, 0x51, 0x03, 0x6b, 0x4b, 0x72, 0x0e, 0xc3, 0x9a, 0x27, 0x92, - 0xa2, 0x94, 0x61, 0x79, 0xcf, 0x68, 0x19, 0x0f, 0x1b, 0x38, 0x4c, 0xac, 0x84, 0x00, 0x75, 0x4d, - 0x94, 0x34, 0xd5, 0x40, 0x64, 0xb3, 0x7a, 0xf3, 0x9b, 0x6f, 0x37, 0x96, 0x7e, 0xf6, 0xed, 0xc6, - 0xd2, 0xcf, 0xbf, 0xdd, 0x48, 0x3c, 0x3b, 0xdf, 0x48, 0x7c, 0x73, 0xbe, 0x91, 0xf8, 0xe9, 0xf9, - 0x46, 0xe2, 0xdf, 0xce, 0x37, 0x12, 0x07, 0x59, 0x11, 0xc2, 0x7c, 0xfc, 0xbf, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x6b, 0x1c, 0x13, 0xe7, 0x66, 0x26, 0x00, 0x00, + 0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0x7b, + 0xbc, 0xe3, 0xf1, 0xfa, 0xcf, 0xb1, 0x35, 0x1f, 0xf0, 0x8e, 0xff, 0x59, 0xbb, 0xf9, 0x21, 0x8b, + 0x6b, 0x89, 0x24, 0x8a, 0x94, 0x9d, 0x41, 0x80, 0x10, 0xa5, 0xee, 0x12, 0xd5, 0xa3, 0x66, 0x37, + 0xd3, 0x5d, 0x94, 0xcc, 0x04, 0x41, 0x8c, 0x1c, 0x92, 0x40, 0xa7, 0xdc, 0x03, 0x21, 0x08, 0x12, + 0xe4, 0x90, 0xc3, 0x5e, 0x72, 0x08, 0x90, 0xd3, 0x20, 0xa7, 0x39, 0x6e, 0x12, 0x20, 0x58, 0x24, + 0x88, 0x91, 0x51, 0xce, 0x01, 0xf6, 0xb2, 0xc8, 0x21, 0x09, 0x10, 0xd4, 0x47, 0x37, 0x9b, 0x32, + 0x2d, 0x7b, 0xb2, 0x7b, 0x21, 0xbb, 0x5e, 0xfd, 0xde, 0xab, 0xaf, 0x57, 0x55, 0xbf, 0xf7, 0x0a, + 0x0a, 0x6c, 0x32, 0xa2, 0x41, 0x65, 0xe4, 0x7b, 0xcc, 0x43, 0xc8, 0xf2, 0xcc, 0x43, 0xea, 0x57, + 0x82, 0x63, 0xe2, 0x0f, 0x0f, 0x6d, 0x56, 0x39, 0xba, 0x57, 0xba, 0xc6, 0xec, 0x21, 0x0d, 0x18, + 0x19, 0x8e, 0x3e, 0x8e, 0xbe, 0x24, 0xbc, 0xf4, 0x8e, 0x35, 0xf6, 0x09, 0xb3, 0x3d, 0xf7, 0xe3, + 0xf0, 0x43, 0x55, 0x5c, 0x19, 0x78, 0x03, 0x4f, 0x7c, 0x7e, 0xcc, 0xbf, 0xa4, 0x54, 0x5f, 0x87, + 0xc5, 0xa7, 0xd4, 0x0f, 0x6c, 0xcf, 0x45, 0x57, 0x20, 0x63, 0xbb, 0x16, 0x7d, 0xbe, 0x9a, 0x28, + 0x27, 0x6e, 0xa5, 0xb1, 0x2c, 0xe8, 0x7f, 0x96, 0x80, 0x82, 0xe1, 0xba, 0x1e, 0x13, 0xb6, 0x02, + 0x84, 0x20, 0xed, 0x92, 0x21, 0x15, 0xa0, 0x3c, 0x16, 0xdf, 0xa8, 0x06, 0x59, 0x87, 0xec, 0x51, + 0x27, 0x58, 0x4d, 0x96, 0x53, 0xb7, 0x0a, 0x1b, 0x3f, 0xa8, 0xbc, 0xda, 0xe7, 0x4a, 0xcc, 0x48, + 0x65, 0x5b, 0xa0, 0x1b, 0x2e, 0xf3, 0x27, 0x58, 0xa9, 0x96, 0x7e, 0x08, 0x85, 0x98, 0x18, 0x69, + 0x90, 0x3a, 0xa4, 0x13, 0xd5, 0x0c, 0xff, 0xe4, 0xfd, 0x3b, 0x22, 0xce, 0x98, 0xae, 0x26, 0x85, + 0x4c, 0x16, 0xbe, 0x48, 0xde, 0x4f, 0xe8, 0x5f, 0x42, 0x1e, 0xd3, 0xc0, 0x1b, 0xfb, 0x26, 0x0d, + 0xd0, 0x47, 0x90, 0x77, 0x89, 0xeb, 0xf5, 0xcd, 0xd1, 0x38, 0x10, 0xea, 0xa9, 0x6a, 0xf1, 0xec, + 0xe5, 0x7a, 0xae, 0x45, 0x5c, 0xaf, 0xd6, 0xd9, 0x0d, 0x70, 0x8e, 0x57, 0xd7, 0x46, 0xe3, 0x00, + 0xbd, 0x0f, 0xc5, 0x21, 0x1d, 0x7a, 0xfe, 0xa4, 0xbf, 0x37, 0x61, 0x34, 0x10, 0x86, 0x53, 0xb8, + 0x20, 0x65, 0x55, 0x2e, 0xd2, 0xff, 0x38, 0x01, 0x57, 0x42, 0xdb, 0x98, 0xfe, 0xd6, 0xd8, 0xf6, + 0xe9, 0x90, 0xba, 0x2c, 0x40, 0x9f, 0x41, 0xd6, 0xb1, 0x87, 0x36, 0x93, 0x6d, 0x14, 0x36, 0xde, + 0x9b, 0x37, 0xe6, 0xa8, 0x57, 0x58, 0x81, 0x91, 0x01, 0x45, 0x9f, 0x06, 0xd4, 0x3f, 0x92, 0x33, + 0x21, 0x9a, 0x7c, 0xa3, 0xf2, 0x8c, 0x8a, 0xbe, 0x09, 0xb9, 0x8e, 0x43, 0xd8, 0xbe, 0xe7, 0x0f, + 0x91, 0x0e, 0x45, 0xe2, 0x9b, 0x07, 0x36, 0xa3, 0x26, 0x1b, 0xfb, 0xe1, 0xaa, 0xcc, 0xc8, 0xd0, + 0x55, 0x48, 0x7a, 0xb2, 0xa1, 0x7c, 0x35, 0x7b, 0xf6, 0x72, 0x3d, 0xd9, 0xee, 0xe2, 0xa4, 0x17, + 0xe8, 0x0f, 0xe0, 0x52, 0xc7, 0x19, 0x0f, 0x6c, 0xb7, 0x4e, 0x03, 0xd3, 0xb7, 0x47, 0xdc, 0x3a, + 0x5f, 0x5e, 0xee, 0x7c, 0xe1, 0xf2, 0xf2, 0xef, 0x68, 0xc9, 0x93, 0xd3, 0x25, 0xd7, 0xff, 0x30, + 0x09, 0x97, 0x1a, 0xee, 0xc0, 0x76, 0x69, 0x5c, 0xfb, 0x26, 0x2c, 0x53, 0x21, 0xec, 0x1f, 0x49, + 0xa7, 0x52, 0x76, 0x96, 0xa4, 0x34, 0xf4, 0xb4, 0xe6, 0x39, 0x7f, 0xb9, 0x37, 0x6f, 0xf8, 0xaf, + 0x58, 0x9f, 0xe7, 0x35, 0xa8, 0x01, 0x8b, 0x23, 0x31, 0x88, 0x60, 0x35, 0x25, 0x6c, 0xdd, 0x9c, + 0x67, 0xeb, 0x95, 0x71, 0x56, 0xd3, 0xdf, 0xbc, 0x5c, 0x5f, 0xc0, 0xa1, 0xee, 0x2f, 0xe3, 0x7c, + 0xff, 0x9e, 0x80, 0x95, 0x96, 0x67, 0xcd, 0xcc, 0x43, 0x09, 0x72, 0x07, 0x5e, 0xc0, 0x62, 0x1b, + 0x25, 0x2a, 0xa3, 0xfb, 0x90, 0x1b, 0xa9, 0xe5, 0x53, 0xab, 0x7f, 0x7d, 0x7e, 0x97, 0x25, 0x06, + 0x47, 0x68, 0xf4, 0x00, 0xf2, 0x7e, 0xe8, 0x13, 0xab, 0xa9, 0xb7, 0x71, 0x9c, 0x29, 0x1e, 0xfd, + 0x1a, 0x64, 0xe5, 0x22, 0xac, 0xa6, 0x85, 0xe6, 0xcd, 0xb7, 0x9a, 0x73, 0xac, 0x94, 0xf4, 0x9f, + 0x25, 0x40, 0xc3, 0x64, 0x9f, 0xed, 0xd0, 0xe1, 0x1e, 0xf5, 0xbb, 0x8c, 0xb0, 0x71, 0x80, 0xae, + 0x42, 0xd6, 0xa1, 0xc4, 0xa2, 0xbe, 0x18, 0x64, 0x0e, 0xab, 0x12, 0xda, 0xe5, 0x4e, 0x4e, 0xcc, + 0x03, 0xb2, 0x67, 0x3b, 0x36, 0x9b, 0x88, 0x61, 0x2e, 0xcf, 0x5f, 0xe5, 0xf3, 0x36, 0x2b, 0x38, + 0xa6, 0x88, 0x67, 0xcc, 0xa0, 0x55, 0x58, 0x1c, 0xd2, 0x20, 0x20, 0x03, 0x2a, 0x46, 0x9f, 0xc7, + 0x61, 0x51, 0x7f, 0x00, 0xc5, 0xb8, 0x1e, 0x2a, 0xc0, 0xe2, 0x6e, 0xeb, 0x49, 0xab, 0xfd, 0xac, + 0xa5, 0x2d, 0xa0, 0x15, 0x28, 0xec, 0xb6, 0x70, 0xc3, 0xa8, 0x6d, 0x19, 0xd5, 0xed, 0x86, 0x96, + 0x40, 0x4b, 0x90, 0x9f, 0x16, 0x93, 0xfa, 0x5f, 0x27, 0x00, 0xf8, 0x02, 0xaa, 0x41, 0x7d, 0x01, + 0x99, 0x80, 0x11, 0x26, 0x17, 0x6e, 0x79, 0xe3, 0x83, 0x79, 0xbd, 0x9e, 0xc2, 0x2b, 0xfc, 0x8f, + 0x62, 0xa9, 0x12, 0xef, 0x61, 0x72, 0xa6, 0x87, 0x7c, 0x0f, 0x11, 0xcb, 0xf2, 0x55, 0xc7, 0xc5, + 0xb7, 0xfe, 0x00, 0x32, 0x42, 0x7b, 0xb6, 0xbb, 0x39, 0x48, 0xd7, 0xf9, 0x57, 0x02, 0xe5, 0x21, + 0x83, 0x1b, 0x46, 0xfd, 0x4b, 0x2d, 0x89, 0x34, 0x28, 0xd6, 0x9b, 0xdd, 0x5a, 0xbb, 0xd5, 0x6a, + 0xd4, 0x7a, 0x8d, 0xba, 0x96, 0xd2, 0x6f, 0x42, 0xa6, 0x39, 0xe4, 0x96, 0xaf, 0x73, 0xaf, 0xd8, + 0xa7, 0x3e, 0x75, 0xcd, 0xd0, 0xd9, 0xa6, 0x02, 0xfd, 0xa7, 0x79, 0xc8, 0xec, 0x78, 0x63, 0x97, + 0xa1, 0x8d, 0xd8, 0xce, 0x5e, 0xde, 0x58, 0x9b, 0x37, 0x2c, 0x01, 0xac, 0xf4, 0x26, 0x23, 0xaa, + 0x76, 0xfe, 0x55, 0xc8, 0x4a, 0xff, 0x51, 0xc3, 0x51, 0x25, 0x2e, 0x67, 0xc4, 0x1f, 0x50, 0xa6, + 0xc6, 0xa3, 0x4a, 0xe8, 0x16, 0xe4, 0x7c, 0x4a, 0x2c, 0xcf, 0x75, 0x26, 0xc2, 0xcd, 0x72, 0xf2, + 0xe8, 0xc5, 0x94, 0x58, 0x6d, 0xd7, 0x99, 0xe0, 0xa8, 0x16, 0x6d, 0x41, 0x71, 0xcf, 0x76, 0xad, + 0xbe, 0x37, 0x92, 0xe7, 0x60, 0xe6, 0xf5, 0x4e, 0x29, 0x7b, 0x55, 0xb5, 0x5d, 0xab, 0x2d, 0xc1, + 0xb8, 0xb0, 0x37, 0x2d, 0xa0, 0x16, 0x2c, 0x1f, 0x79, 0xce, 0x78, 0x48, 0x23, 0x5b, 0x59, 0x61, + 0xeb, 0xc3, 0xd7, 0xdb, 0x7a, 0x2a, 0xf0, 0xa1, 0xb5, 0xa5, 0xa3, 0x78, 0x11, 0x3d, 0x81, 0x25, + 0x36, 0x1c, 0xed, 0x07, 0x91, 0xb9, 0x45, 0x61, 0xee, 0xfb, 0x17, 0x4c, 0x18, 0x87, 0x87, 0xd6, + 0x8a, 0x2c, 0x56, 0x2a, 0xfd, 0x7e, 0x0a, 0x0a, 0xb1, 0x9e, 0xa3, 0x2e, 0x14, 0x46, 0xbe, 0x37, + 0x22, 0x03, 0x71, 0x96, 0xab, 0xb5, 0xb8, 0xf7, 0x56, 0xa3, 0xae, 0x74, 0xa6, 0x8a, 0x38, 0x6e, + 0x45, 0x3f, 0x4d, 0x42, 0x21, 0x56, 0x89, 0x6e, 0x43, 0x0e, 0x77, 0x70, 0xf3, 0xa9, 0xd1, 0x6b, + 0x68, 0x0b, 0xa5, 0xeb, 0x27, 0xa7, 0xe5, 0x55, 0x61, 0x2d, 0x6e, 0xa0, 0xe3, 0xdb, 0x47, 0xdc, + 0xf5, 0x6e, 0xc1, 0x62, 0x08, 0x4d, 0x94, 0xde, 0x3d, 0x39, 0x2d, 0xbf, 0x73, 0x1e, 0x1a, 0x43, + 0xe2, 0xee, 0x96, 0x81, 0x1b, 0x75, 0x2d, 0x39, 0x1f, 0x89, 0xbb, 0x07, 0xc4, 0xa7, 0x16, 0xfa, + 0x3e, 0x64, 0x15, 0x30, 0x55, 0x2a, 0x9d, 0x9c, 0x96, 0xaf, 0x9e, 0x07, 0x4e, 0x71, 0xb8, 0xbb, + 0x6d, 0x3c, 0x6d, 0x68, 0xe9, 0xf9, 0x38, 0xdc, 0x75, 0xc8, 0x11, 0x45, 0x1f, 0x40, 0x46, 0xc2, + 0x32, 0xa5, 0x6b, 0x27, 0xa7, 0xe5, 0xef, 0xbd, 0x62, 0x8e, 0xa3, 0x4a, 0xab, 0x7f, 0xf4, 0xe7, + 0x6b, 0x0b, 0x7f, 0xfb, 0x17, 0x6b, 0xda, 0xf9, 0xea, 0xd2, 0x7f, 0x27, 0x60, 0x69, 0x66, 0xc9, + 0x91, 0x0e, 0x59, 0xd7, 0x33, 0xbd, 0x91, 0x3c, 0xe2, 0x73, 0x55, 0x38, 0x7b, 0xb9, 0x9e, 0x6d, + 0x79, 0x35, 0x6f, 0x34, 0xc1, 0xaa, 0x06, 0x3d, 0x39, 0x77, 0x49, 0x7d, 0xf2, 0x96, 0xfe, 0x34, + 0xf7, 0x9a, 0x7a, 0x08, 0x4b, 0x96, 0x6f, 0x1f, 0x51, 0xbf, 0x6f, 0x7a, 0xee, 0xbe, 0x3d, 0x50, + 0xc7, 0x77, 0x69, 0x9e, 0xcd, 0xba, 0x00, 0xe2, 0xa2, 0x54, 0xa8, 0x09, 0xfc, 0x2f, 0x71, 0x41, + 0x95, 0x9e, 0x42, 0x31, 0xee, 0xa1, 0xe8, 0x3d, 0x80, 0xc0, 0xfe, 0x6d, 0xaa, 0x38, 0x8f, 0x60, + 0x48, 0x38, 0xcf, 0x25, 0x82, 0xf1, 0xa0, 0x0f, 0x21, 0x3d, 0xf4, 0x2c, 0x69, 0x67, 0xa9, 0x7a, + 0x99, 0xdf, 0x93, 0xff, 0xfc, 0x72, 0xbd, 0xe0, 0x05, 0x95, 0x4d, 0xdb, 0xa1, 0x3b, 0x9e, 0x45, + 0xb1, 0x00, 0xe8, 0x47, 0x90, 0xe6, 0x47, 0x05, 0x7a, 0x17, 0xd2, 0xd5, 0x66, 0xab, 0xae, 0x2d, + 0x94, 0x2e, 0x9d, 0x9c, 0x96, 0x97, 0xc4, 0x94, 0xf0, 0x0a, 0xee, 0xbb, 0x68, 0x1d, 0xb2, 0x4f, + 0xdb, 0xdb, 0xbb, 0x3b, 0xdc, 0xbd, 0x2e, 0x9f, 0x9c, 0x96, 0x57, 0xa2, 0x6a, 0x39, 0x69, 0xe8, + 0x3d, 0xc8, 0xf4, 0x76, 0x3a, 0x9b, 0x5d, 0x2d, 0x59, 0x42, 0x27, 0xa7, 0xe5, 0xe5, 0xa8, 0x5e, + 0xf4, 0xb9, 0x74, 0x49, 0xad, 0x6a, 0x3e, 0x92, 0xeb, 0xff, 0x95, 0x84, 0x25, 0xcc, 0x39, 0xaf, + 0xcf, 0x3a, 0x9e, 0x63, 0x9b, 0x13, 0xd4, 0x81, 0xbc, 0xe9, 0xb9, 0x96, 0x1d, 0xdb, 0x53, 0x1b, + 0xaf, 0xb9, 0x18, 0xa7, 0x5a, 0x61, 0xa9, 0x16, 0x6a, 0xe2, 0xa9, 0x11, 0xb4, 0x01, 0x19, 0x8b, + 0x3a, 0x64, 0x72, 0xd1, 0x0d, 0x5d, 0x57, 0xfc, 0x1a, 0x4b, 0xa8, 0x60, 0x93, 0xe4, 0x79, 0x9f, + 0x30, 0x46, 0x87, 0x23, 0x26, 0x6f, 0xe8, 0x34, 0x2e, 0x0c, 0xc9, 0x73, 0x43, 0x89, 0xd0, 0xa7, + 0x90, 0x3d, 0xb6, 0x5d, 0xcb, 0x3b, 0x56, 0x97, 0xf0, 0xc5, 0x76, 0x15, 0x56, 0x3f, 0xe1, 0x77, + 0xef, 0xb9, 0xce, 0xf2, 0x59, 0x6f, 0xb5, 0x5b, 0x8d, 0x70, 0xd6, 0x55, 0x7d, 0xdb, 0x6d, 0x79, + 0x2e, 0xdf, 0x31, 0xd0, 0x6e, 0xf5, 0x37, 0x8d, 0xe6, 0xf6, 0x2e, 0xe6, 0x33, 0x7f, 0xe5, 0xe4, + 0xb4, 0xac, 0x45, 0x90, 0x4d, 0x62, 0x3b, 0x9c, 0x18, 0x5e, 0x83, 0x94, 0xd1, 0xfa, 0x52, 0x4b, + 0x96, 0xb4, 0x93, 0xd3, 0x72, 0x31, 0xaa, 0x36, 0xdc, 0xc9, 0x74, 0x33, 0x9d, 0x6f, 0x57, 0xff, + 0xd7, 0x24, 0x14, 0x77, 0x47, 0x16, 0x61, 0x54, 0x7a, 0x26, 0x2a, 0x43, 0x61, 0x44, 0x7c, 0xe2, + 0x38, 0xd4, 0xb1, 0x83, 0xa1, 0x0a, 0x1e, 0xe2, 0x22, 0x74, 0xff, 0x3b, 0x4c, 0xa6, 0x22, 0x66, + 0x6a, 0x4a, 0x77, 0x61, 0x79, 0x5f, 0x76, 0xb6, 0x4f, 0x4c, 0xb1, 0xba, 0x29, 0xb1, 0xba, 0x95, + 0x79, 0x26, 0xe2, 0xbd, 0xaa, 0xa8, 0x31, 0x1a, 0x42, 0x0b, 0x2f, 0xed, 0xc7, 0x8b, 0xe8, 0x73, + 0x58, 0x1c, 0x7a, 0xae, 0xcd, 0x3c, 0xff, 0xad, 0xd6, 0x21, 0x04, 0xa3, 0xdb, 0x70, 0x89, 0xaf, + 0x70, 0xd8, 0x25, 0x51, 0x2d, 0x6e, 0xae, 0x24, 0x5e, 0x19, 0x92, 0xe7, 0xaa, 0x4d, 0xcc, 0xc5, + 0xfa, 0xe7, 0xb0, 0x34, 0xd3, 0x07, 0x7e, 0x9b, 0x77, 0x8c, 0xdd, 0x6e, 0x43, 0x5b, 0x40, 0x45, + 0xc8, 0xd5, 0xda, 0xad, 0x5e, 0xb3, 0xb5, 0xcb, 0xe9, 0x48, 0x11, 0x72, 0xb8, 0xbd, 0xbd, 0x5d, + 0x35, 0x6a, 0x4f, 0xb4, 0xa4, 0xfe, 0x8b, 0x68, 0x7e, 0x15, 0x1f, 0xa9, 0xce, 0xf2, 0x91, 0x3b, + 0xaf, 0x1f, 0xba, 0x62, 0x24, 0xd3, 0x42, 0xc4, 0x4b, 0xfe, 0x3f, 0x80, 0x58, 0x46, 0x6a, 0xf5, + 0x09, 0xbb, 0x28, 0xe6, 0xe8, 0x85, 0xd1, 0x24, 0xce, 0x2b, 0x05, 0x83, 0xa1, 0x47, 0x50, 0x34, + 0xbd, 0xe1, 0xc8, 0xa1, 0x4a, 0x3f, 0xf5, 0x36, 0xfa, 0x85, 0x48, 0xc5, 0x60, 0x71, 0x5e, 0x94, + 0x9e, 0x65, 0x6e, 0x7f, 0x90, 0x80, 0x42, 0xac, 0xc3, 0xb3, 0x54, 0xa8, 0x08, 0xb9, 0xdd, 0x4e, + 0xdd, 0xe8, 0x35, 0x5b, 0x8f, 0xb5, 0x04, 0x02, 0xc8, 0x8a, 0x09, 0xac, 0x6b, 0x49, 0x4e, 0xe1, + 0x6a, 0xed, 0x9d, 0xce, 0x76, 0x43, 0x90, 0x21, 0x74, 0x05, 0xb4, 0x70, 0x0a, 0xfb, 0xdd, 0x9e, + 0x81, 0xb9, 0x34, 0x8d, 0x2e, 0xc3, 0x4a, 0x24, 0x55, 0x9a, 0x19, 0x74, 0x15, 0x50, 0x24, 0x9c, + 0x9a, 0xc8, 0xea, 0xbf, 0x0b, 0x2b, 0x35, 0xcf, 0x65, 0xc4, 0x76, 0x23, 0x7a, 0xbb, 0xc1, 0xc7, + 0xad, 0x44, 0x7d, 0xdb, 0x92, 0xa7, 0x6d, 0x75, 0xe5, 0xec, 0xe5, 0x7a, 0x21, 0x82, 0x36, 0xeb, + 0x7c, 0xa4, 0x61, 0xc1, 0xe2, 0x7b, 0x6a, 0x64, 0x5b, 0x62, 0x8a, 0x33, 0xd5, 0xc5, 0xb3, 0x97, + 0xeb, 0xa9, 0x4e, 0xb3, 0x8e, 0xb9, 0x0c, 0xbd, 0x0b, 0x79, 0xfa, 0xdc, 0x66, 0x7d, 0x93, 0x9f, + 0xae, 0x7c, 0x0e, 0x33, 0x38, 0xc7, 0x05, 0x35, 0x7e, 0x98, 0x56, 0x01, 0x3a, 0x9e, 0xcf, 0x54, + 0xcb, 0x9f, 0x42, 0x66, 0xe4, 0xf9, 0x22, 0xb6, 0xe4, 0x57, 0xcf, 0x5c, 0xb2, 0xc6, 0xe1, 0xd2, + 0xd9, 0xb1, 0x04, 0xeb, 0x7f, 0x97, 0x04, 0xe8, 0x91, 0xe0, 0x50, 0x19, 0x79, 0x00, 0xf9, 0x28, + 0x39, 0x70, 0x51, 0x90, 0x1a, 0x5b, 0xf3, 0x08, 0x8f, 0x3e, 0x09, 0xbd, 0x4e, 0x72, 0xf7, 0xf9, + 0x8a, 0xaa, 0xad, 0x79, 0xf4, 0x77, 0x96, 0xa0, 0xf3, 0xfb, 0x8a, 0xfa, 0xbe, 0x5a, 0x7c, 0xfe, + 0x89, 0x6a, 0xe2, 0xcc, 0x96, 0xf3, 0xa6, 0xd8, 0xdf, 0x8d, 0x79, 0x8d, 0x9c, 0x5b, 0x94, 0xad, + 0x05, 0x3c, 0xd5, 0x43, 0x0f, 0xa1, 0xc0, 0x87, 0xde, 0x0f, 0x44, 0x9d, 0x22, 0x7e, 0xaf, 0x9d, + 0x2d, 0x69, 0x01, 0xc3, 0x28, 0xfa, 0xae, 0x6a, 0xb0, 0xec, 0x8f, 0x5d, 0x3e, 0x6c, 0x65, 0x43, + 0xb7, 0xe1, 0x9d, 0x16, 0x65, 0xc7, 0x9e, 0x7f, 0x68, 0x30, 0x46, 0xcc, 0x03, 0x1e, 0xed, 0xab, + 0x93, 0x6e, 0xca, 0x7a, 0x13, 0x33, 0xac, 0x77, 0x15, 0x16, 0x89, 0x63, 0x93, 0x80, 0x4a, 0xaa, + 0x90, 0xc7, 0x61, 0x91, 0x73, 0x73, 0xce, 0xf4, 0x69, 0x10, 0x50, 0x19, 0x9f, 0xe6, 0xf1, 0x54, + 0xa0, 0xff, 0x63, 0x12, 0xa0, 0xd9, 0x31, 0x76, 0x94, 0xf9, 0x3a, 0x64, 0xf7, 0xc9, 0xd0, 0x76, + 0x26, 0x17, 0xed, 0xf4, 0x29, 0xbe, 0x62, 0x48, 0x43, 0x9b, 0x42, 0x07, 0x2b, 0x5d, 0x41, 0xd9, + 0xc7, 0x7b, 0x2e, 0x65, 0x11, 0x65, 0x17, 0x25, 0xce, 0x0f, 0x7c, 0xe2, 0x46, 0x2b, 0x23, 0x0b, + 0xbc, 0xeb, 0x03, 0xc2, 0xe8, 0x31, 0x99, 0x84, 0x1b, 0x53, 0x15, 0xd1, 0x16, 0xa7, 0xf2, 0x01, + 0xf5, 0x8f, 0xa8, 0xb5, 0x9a, 0x11, 0x5e, 0xf8, 0xa6, 0xfe, 0x60, 0x05, 0x97, 0xcc, 0x27, 0xd2, + 0x2e, 0x3d, 0x10, 0xd7, 0xf5, 0xb4, 0xea, 0x3b, 0x45, 0xd7, 0x77, 0x61, 0x69, 0x66, 0x9c, 0xaf, + 0xc4, 0x4a, 0xcd, 0xce, 0xd3, 0x4f, 0xb5, 0xb4, 0xfa, 0xfa, 0x5c, 0xcb, 0xea, 0x7f, 0x95, 0x92, + 0x5b, 0x49, 0xcd, 0xea, 0xfc, 0x7c, 0x55, 0x4e, 0x64, 0xbf, 0x4c, 0xcf, 0x51, 0xfe, 0xfd, 0xe1, + 0xc5, 0x3b, 0x8c, 0x73, 0x6f, 0x01, 0xc7, 0x91, 0x22, 0x5a, 0x87, 0x82, 0x5c, 0xff, 0x3e, 0xf7, + 0x27, 0x31, 0xad, 0x4b, 0x18, 0xa4, 0x88, 0x6b, 0xa2, 0x9b, 0xb0, 0x3c, 0x1a, 0xef, 0x39, 0x76, + 0x70, 0x40, 0x2d, 0x89, 0x49, 0x0b, 0xcc, 0x52, 0x24, 0x15, 0xb0, 0x1d, 0x28, 0x2a, 0x41, 0x5f, + 0xf0, 0xae, 0x8c, 0xe8, 0xd0, 0xed, 0x37, 0x75, 0x48, 0xaa, 0x08, 0x3a, 0x56, 0x18, 0x4d, 0x0b, + 0x7a, 0x1d, 0x72, 0x61, 0x67, 0xd1, 0x2a, 0xa4, 0x7a, 0xb5, 0x8e, 0xb6, 0x50, 0x5a, 0x39, 0x39, + 0x2d, 0x17, 0x42, 0x71, 0xaf, 0xd6, 0xe1, 0x35, 0xbb, 0xf5, 0x8e, 0x96, 0x98, 0xad, 0xd9, 0xad, + 0x77, 0x4a, 0x69, 0x7e, 0xf3, 0xeb, 0xfb, 0x50, 0x88, 0xb5, 0x80, 0x6e, 0xc0, 0x62, 0xb3, 0xf5, + 0x18, 0x37, 0xba, 0x5d, 0x6d, 0xa1, 0x74, 0xf5, 0xe4, 0xb4, 0x8c, 0x62, 0xb5, 0x4d, 0x77, 0xc0, + 0xd7, 0x07, 0xbd, 0x07, 0xe9, 0xad, 0x76, 0xb7, 0x17, 0x12, 0xbd, 0x18, 0x62, 0xcb, 0x0b, 0x58, + 0xe9, 0xb2, 0xa2, 0x14, 0x71, 0xc3, 0xfa, 0x9f, 0x24, 0x20, 0x2b, 0xf9, 0xee, 0xdc, 0x85, 0x32, + 0x60, 0x31, 0x8c, 0xc2, 0x24, 0x09, 0xff, 0xf0, 0xf5, 0x84, 0xb9, 0xa2, 0xf8, 0xad, 0x74, 0xbf, + 0x50, 0xaf, 0xf4, 0x05, 0x14, 0xe3, 0x15, 0xdf, 0xc9, 0xf9, 0x7e, 0x07, 0x0a, 0xdc, 0xbf, 0x43, + 0xe2, 0xbc, 0x01, 0x59, 0xc9, 0xc9, 0xd5, 0x69, 0x7a, 0x11, 0x7b, 0x57, 0x48, 0x74, 0x1f, 0x16, + 0x25, 0xe3, 0x0f, 0xf3, 0x53, 0x6b, 0x17, 0xef, 0x22, 0x1c, 0xc2, 0xf5, 0x87, 0x90, 0xee, 0x50, + 0xea, 0xf3, 0xb9, 0x77, 0x3d, 0x8b, 0x4e, 0x2f, 0x20, 0x15, 0xac, 0x58, 0xb4, 0x59, 0xe7, 0xc1, + 0x8a, 0x45, 0x9b, 0x56, 0x94, 0x5e, 0x48, 0xc6, 0xd2, 0x0b, 0x3d, 0x28, 0x3e, 0xa3, 0xf6, 0xe0, + 0x80, 0x51, 0x4b, 0x18, 0xba, 0x03, 0xe9, 0x11, 0x8d, 0x3a, 0xbf, 0x3a, 0xd7, 0xc1, 0x28, 0xf5, + 0xb1, 0x40, 0xf1, 0x73, 0xe4, 0x58, 0x68, 0xab, 0xac, 0xa8, 0x2a, 0xe9, 0xff, 0x90, 0x84, 0xe5, + 0x66, 0x10, 0x8c, 0x89, 0x6b, 0x86, 0x0c, 0xe5, 0x47, 0xb3, 0x0c, 0xe5, 0xd6, 0xdc, 0x11, 0xce, + 0xa8, 0xcc, 0x66, 0x4d, 0xd4, 0xe5, 0x90, 0x8c, 0x2e, 0x07, 0xfd, 0x3f, 0x12, 0x61, 0x6a, 0xe4, + 0x66, 0x6c, 0xbb, 0x97, 0x56, 0x4f, 0x4e, 0xcb, 0x57, 0xe2, 0x96, 0xe8, 0xae, 0x7b, 0xe8, 0x7a, + 0xc7, 0x2e, 0x7a, 0x1f, 0x32, 0xb8, 0xd1, 0x6a, 0x3c, 0xd3, 0x12, 0xd2, 0x3d, 0x67, 0x40, 0x98, + 0xba, 0xf4, 0x98, 0x5b, 0xea, 0x34, 0x5a, 0x75, 0xce, 0x25, 0x92, 0x73, 0x2c, 0x75, 0xa8, 0x6b, + 0xd9, 0xee, 0x00, 0xdd, 0x80, 0x6c, 0xb3, 0xdb, 0xdd, 0x15, 0xc1, 0xeb, 0x3b, 0x27, 0xa7, 0xe5, + 0xcb, 0x33, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x1c, 0x10, 0xe7, 0x7d, + 0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0xe6, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb, 0x97, + 0x24, 0x68, 0x86, 0x69, 0xd2, 0x11, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xc4, 0xbf, 0x6c, + 0x1a, 0xf2, 0x80, 0xfb, 0x73, 0xf3, 0xea, 0xe7, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x0d, 0xed, + 0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0x3c, 0x07, 0x81, 0xee, + 0x42, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0xaf, 0xbf, 0x2e, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2, 0x35, + 0x00, 0x32, 0x66, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x83, 0x6c, 0x40, + 0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xe1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9, 0x52, + 0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde, 0x44, + 0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x67, 0xd4, 0x77, 0x89, + 0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0x34, 0x37, 0x17, 0x1a, 0x69, 0x54, 0x6a, + 0xc6, 0x9c, 0xf3, 0xff, 0x1a, 0xa4, 0xc6, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe2, 0x6d, + 0xcc, 0x65, 0xa8, 0x31, 0x3d, 0xb6, 0x52, 0xaf, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f, 0xba, + 0xee, 0x00, 0x4c, 0x7b, 0x8d, 0xd6, 0x20, 0x53, 0xdb, 0xec, 0x76, 0xb7, 0xb5, 0x05, 0x79, 0x36, + 0x4f, 0xab, 0x84, 0x58, 0xff, 0xcb, 0x04, 0xe4, 0x6a, 0x86, 0xba, 0x31, 0x37, 0x41, 0x13, 0x07, + 0x8e, 0x49, 0x7d, 0xd6, 0xa7, 0xcf, 0x47, 0xb6, 0x3f, 0x51, 0x67, 0xc6, 0xc5, 0x61, 0xd2, 0x32, + 0xd7, 0xaa, 0x51, 0x9f, 0x35, 0x84, 0x0e, 0xc2, 0x50, 0xa4, 0x6a, 0x88, 0x7d, 0x93, 0x84, 0x27, + 0xf8, 0xda, 0xc5, 0x53, 0x21, 0xe9, 0xf5, 0xb4, 0x1c, 0xe0, 0x42, 0x68, 0xa4, 0x46, 0x02, 0xfd, + 0x29, 0x5c, 0x6e, 0xfb, 0xe6, 0x01, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x21, 0x5c, 0x67, 0x24, + 0x38, 0xec, 0x1f, 0xd8, 0x01, 0xf3, 0xfc, 0x49, 0xdf, 0xa7, 0x8c, 0xba, 0xbc, 0xbe, 0x2f, 0x9e, + 0x5d, 0x54, 0x92, 0xe3, 0x1a, 0xc7, 0x6c, 0x49, 0x08, 0x0e, 0x11, 0xdb, 0x1c, 0xa0, 0x37, 0xa1, + 0xc8, 0xd9, 0x6c, 0x9d, 0xee, 0x93, 0xb1, 0xc3, 0x02, 0xf4, 0x43, 0x00, 0xc7, 0x1b, 0xf4, 0xdf, + 0xfa, 0xb8, 0xcf, 0x3b, 0xde, 0x40, 0x7e, 0xea, 0xbf, 0x01, 0x5a, 0xdd, 0x0e, 0x46, 0x84, 0x99, + 0x07, 0x61, 0xf6, 0x06, 0x3d, 0x06, 0xed, 0x80, 0x12, 0x9f, 0xed, 0x51, 0xc2, 0xfa, 0x23, 0xea, + 0xdb, 0x9e, 0xf5, 0x56, 0x53, 0xba, 0x12, 0x69, 0x75, 0x84, 0x92, 0xfe, 0x9f, 0x09, 0x00, 0x4c, + 0xf6, 0x43, 0x72, 0xf3, 0x03, 0xb8, 0x14, 0xb8, 0x64, 0x14, 0x1c, 0x78, 0xac, 0x6f, 0xbb, 0x8c, + 0xfa, 0x47, 0xc4, 0x51, 0x11, 0xb8, 0x16, 0x56, 0x34, 0x95, 0x1c, 0xdd, 0x01, 0x74, 0x48, 0xe9, + 0xa8, 0xef, 0x39, 0x56, 0x3f, 0xac, 0x94, 0xef, 0x42, 0x69, 0xac, 0xf1, 0x9a, 0xb6, 0x63, 0x75, + 0x43, 0x39, 0xaa, 0xc2, 0x1a, 0x9f, 0x01, 0xea, 0x32, 0xdf, 0xa6, 0x41, 0x7f, 0xdf, 0xf3, 0xfb, + 0x81, 0xe3, 0x1d, 0xf7, 0xf7, 0x3d, 0xc7, 0xf1, 0x8e, 0xa9, 0x1f, 0xe6, 0x37, 0x4a, 0x8e, 0x37, + 0x68, 0x48, 0xd0, 0xa6, 0xe7, 0x77, 0x1d, 0xef, 0x78, 0x33, 0x44, 0x70, 0x06, 0x34, 0x1d, 0x36, + 0xb3, 0xcd, 0xc3, 0x90, 0x01, 0x45, 0xd2, 0x9e, 0x6d, 0x1e, 0xa2, 0x1b, 0xb0, 0x44, 0x1d, 0x2a, + 0xa2, 0x64, 0x89, 0xca, 0x08, 0x54, 0x31, 0x14, 0x72, 0x90, 0xfe, 0x08, 0xb4, 0x86, 0x6b, 0xfa, + 0x93, 0x51, 0x6c, 0xd9, 0xef, 0x00, 0xe2, 0xe7, 0x4d, 0xdf, 0xf1, 0xcc, 0xc3, 0xfe, 0x90, 0xb8, + 0x64, 0xc0, 0xfb, 0x25, 0xdf, 0x22, 0x34, 0x5e, 0xb3, 0xed, 0x99, 0x87, 0x3b, 0x4a, 0xae, 0xff, + 0x3f, 0xc8, 0x77, 0x1c, 0x62, 0x8a, 0xf7, 0x3b, 0x54, 0x06, 0x1e, 0xb6, 0x71, 0x37, 0xb2, 0x5d, + 0x15, 0x67, 0xe5, 0x71, 0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x43, 0xea, + 0x8a, 0xa7, 0x0e, 0x1e, 0x13, 0x28, 0x67, 0xc8, 0x63, 0x55, 0x12, 0x21, 0x8f, 0x6c, 0x20, 0xca, + 0xf8, 0xcb, 0xa2, 0xfe, 0x4d, 0x02, 0xb2, 0xd8, 0xf3, 0x58, 0xcd, 0x40, 0x65, 0xc8, 0x9a, 0xa4, + 0x1f, 0xee, 0xdd, 0x62, 0x35, 0x7f, 0xf6, 0x72, 0x3d, 0x53, 0x33, 0x9e, 0xd0, 0x09, 0xce, 0x98, + 0xe4, 0x09, 0x9d, 0xf0, 0x4b, 0xde, 0x24, 0x62, 0xc7, 0x09, 0x33, 0x45, 0x79, 0xc9, 0xd7, 0x0c, + 0xbe, 0x9d, 0x70, 0xd6, 0x24, 0xfc, 0x1f, 0xdd, 0x85, 0xa2, 0x02, 0xf5, 0x0f, 0x48, 0x70, 0x20, + 0x99, 0x7c, 0x75, 0xf9, 0xec, 0xe5, 0x3a, 0x48, 0xe4, 0x16, 0x09, 0x0e, 0x30, 0x48, 0x34, 0xff, + 0x46, 0x0d, 0x28, 0x7c, 0xe5, 0xd9, 0x6e, 0x9f, 0x89, 0x41, 0xa8, 0x64, 0xc7, 0xdc, 0x1d, 0x38, + 0x1d, 0xaa, 0xca, 0xc0, 0xc0, 0x57, 0x91, 0x44, 0xff, 0xa7, 0x04, 0x14, 0xb8, 0x4d, 0x7b, 0xdf, + 0x36, 0xf9, 0xa5, 0xfc, 0xdd, 0xef, 0x8a, 0x6b, 0x90, 0x32, 0x03, 0x5f, 0x8d, 0x4d, 0x1c, 0x96, + 0xb5, 0x2e, 0xc6, 0x5c, 0x86, 0x1e, 0x41, 0x56, 0x85, 0x6f, 0xf2, 0x9a, 0xd0, 0xdf, 0x4c, 0x1f, + 0x54, 0x17, 0x95, 0x9e, 0x58, 0xcb, 0x69, 0xef, 0xc4, 0x28, 0x8b, 0x38, 0x2e, 0x42, 0x57, 0x21, + 0x69, 0xba, 0xc2, 0xad, 0xd4, 0x13, 0x68, 0xad, 0x85, 0x93, 0xa6, 0xab, 0xff, 0x7d, 0x02, 0x96, + 0xa6, 0x5e, 0xc5, 0x17, 0xe2, 0x3a, 0xe4, 0x83, 0xf1, 0x5e, 0x30, 0x09, 0x18, 0x1d, 0x86, 0xaf, + 0x29, 0x91, 0x00, 0x35, 0x21, 0x4f, 0x9c, 0x81, 0xe7, 0xdb, 0xec, 0x60, 0xa8, 0x22, 0x87, 0xf9, + 0x47, 0x7b, 0xdc, 0x66, 0xc5, 0x08, 0x55, 0xf0, 0x54, 0x3b, 0x3c, 0xcc, 0x53, 0xa2, 0xb3, 0xe2, + 0x30, 0x7f, 0x1f, 0x8a, 0x0e, 0x19, 0x8a, 0x78, 0x96, 0x07, 0xa4, 0x62, 0x1c, 0x69, 0x5c, 0x50, + 0x32, 0x1e, 0xa5, 0xeb, 0x3a, 0xe4, 0x23, 0x63, 0x68, 0x05, 0x0a, 0x46, 0xa3, 0xdb, 0xbf, 0xb7, + 0x71, 0xbf, 0xff, 0xb8, 0xb6, 0xa3, 0x2d, 0x28, 0x2e, 0xf1, 0x37, 0x09, 0x58, 0x52, 0x3e, 0xaf, + 0xf8, 0xd9, 0x0d, 0x58, 0xf4, 0xc9, 0x3e, 0x0b, 0x19, 0x64, 0x5a, 0x3a, 0x17, 0x3f, 0x46, 0x38, + 0x83, 0xe4, 0x55, 0xf3, 0x19, 0x64, 0xec, 0x7d, 0x2f, 0x75, 0xe1, 0xfb, 0x5e, 0xfa, 0x57, 0xf2, + 0xbe, 0xa7, 0xff, 0x24, 0x09, 0x2b, 0xea, 0xaa, 0x0f, 0xdf, 0xaf, 0xd0, 0x47, 0x90, 0x97, 0xb7, + 0xfe, 0x94, 0xff, 0x8a, 0x27, 0x25, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37, 0x2d, 0x1e, 0x90, + 0x29, 0x68, 0xec, 0xb5, 0x1a, 0xa4, 0xa8, 0xc5, 0xa3, 0x89, 0x3a, 0xa4, 0xf7, 0x6d, 0x87, 0x2a, + 0x3f, 0x9b, 0x9b, 0x43, 0x3c, 0xd7, 0xbc, 0x48, 0x79, 0xf7, 0x44, 0x48, 0xb7, 0xb5, 0x80, 0x85, + 0x76, 0xe9, 0xf7, 0x00, 0xa6, 0xd2, 0xb9, 0x51, 0x0b, 0x67, 0x06, 0x2a, 0x07, 0x14, 0x32, 0x83, + 0x66, 0x1d, 0x73, 0x19, 0xaf, 0x1a, 0xd8, 0x96, 0xda, 0xb9, 0xa2, 0xea, 0x31, 0xaf, 0x1a, 0xd8, + 0x56, 0x94, 0x77, 0x4f, 0xbf, 0x21, 0xef, 0x5e, 0xcd, 0x85, 0x69, 0x08, 0xbd, 0x0d, 0x57, 0xab, + 0x0e, 0x31, 0x0f, 0x1d, 0x3b, 0x60, 0xd4, 0x8a, 0xef, 0xd0, 0xcf, 0x20, 0x3b, 0x73, 0x73, 0xbf, + 0x21, 0xf1, 0xa3, 0xc0, 0xfa, 0x4f, 0x12, 0x50, 0xdc, 0xa2, 0xc4, 0x61, 0x07, 0xd3, 0xe8, 0x99, + 0xd1, 0x80, 0xa9, 0xf3, 0x51, 0x7c, 0xa3, 0xfb, 0x90, 0x8b, 0xee, 0x9a, 0xb7, 0x49, 0x8f, 0x47, + 0x68, 0xf4, 0x39, 0x2c, 0x72, 0xcf, 0xf6, 0xc6, 0x21, 0x25, 0x7c, 0x43, 0xde, 0x55, 0x81, 0xf9, + 0x21, 0xeb, 0x53, 0x71, 0xc5, 0x88, 0xd9, 0xc9, 0xe0, 0xb0, 0xa8, 0xff, 0x4f, 0x02, 0xae, 0xec, + 0x90, 0xc9, 0x1e, 0x55, 0x3b, 0x8e, 0x5a, 0x98, 0x9a, 0x9e, 0x6f, 0xa1, 0x4e, 0x7c, 0xa7, 0x5e, + 0xf0, 0x24, 0x30, 0x4f, 0x79, 0xfe, 0x86, 0x0d, 0xb9, 0x66, 0x32, 0xc6, 0x35, 0xaf, 0x40, 0xc6, + 0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28, 0x4f, 0x2f, 0xb2, + 0xec, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x47, 0x50, 0xea, 0x36, 0x6a, 0xb8, 0xd1, 0xab, 0xb6, 0x7f, + 0xbd, 0xdf, 0x35, 0xb6, 0xbb, 0xc6, 0xc6, 0xdd, 0x7e, 0xa7, 0xbd, 0xfd, 0xe5, 0xbd, 0x4f, 0xee, + 0x7e, 0xa6, 0x25, 0x4a, 0xe5, 0x93, 0xd3, 0xf2, 0xf5, 0x96, 0x51, 0xdb, 0x96, 0x6e, 0xb9, 0xe7, + 0x3d, 0xef, 0x12, 0x27, 0x20, 0x1b, 0x77, 0x3b, 0x9e, 0x33, 0xe1, 0x98, 0xdb, 0xbf, 0x48, 0x41, + 0x3e, 0x4a, 0xc3, 0x71, 0xef, 0xe2, 0x31, 0x90, 0x6a, 0x2a, 0x92, 0xb7, 0xe8, 0x31, 0x7a, 0x7f, + 0x1a, 0xfd, 0x3c, 0x92, 0xcf, 0x01, 0x51, 0x75, 0x18, 0xf9, 0x7c, 0x00, 0x39, 0xa3, 0xdb, 0x6d, + 0x3e, 0x6e, 0x35, 0xea, 0xda, 0xd7, 0x89, 0xd2, 0xf7, 0x4e, 0x4e, 0xcb, 0x97, 0x22, 0x90, 0x11, + 0x04, 0xf6, 0xc0, 0xa5, 0x96, 0x40, 0xd5, 0x6a, 0x8d, 0x4e, 0xaf, 0x51, 0xd7, 0x5e, 0x24, 0xcf, + 0xa3, 0x04, 0x9b, 0x17, 0x4f, 0x7b, 0xf9, 0x0e, 0x6e, 0x74, 0x0c, 0xcc, 0x1b, 0xfc, 0x3a, 0x29, + 0x83, 0xb2, 0x69, 0x8b, 0x3e, 0x1d, 0x11, 0x9f, 0xb7, 0xb9, 0x16, 0x3e, 0x71, 0xbf, 0x48, 0xc9, + 0xe7, 0x9f, 0x69, 0x4e, 0x91, 0x12, 0x6b, 0xc2, 0x5b, 0x13, 0xf9, 0x5c, 0x61, 0x26, 0x75, 0xae, + 0xb5, 0x2e, 0x23, 0x3e, 0xe3, 0x56, 0x74, 0x58, 0xc4, 0xbb, 0xad, 0x16, 0x07, 0xbd, 0x48, 0x9f, + 0x1b, 0x1d, 0x1e, 0xbb, 0x2e, 0xc7, 0xdc, 0x84, 0x5c, 0x98, 0xee, 0xd5, 0xbe, 0x4e, 0x9f, 0xeb, + 0x50, 0x2d, 0xcc, 0x55, 0x8b, 0x06, 0xb7, 0x76, 0x7b, 0xe2, 0x05, 0xfe, 0x45, 0xe6, 0x7c, 0x83, + 0x07, 0x63, 0x66, 0xf1, 0x70, 0xb3, 0x1c, 0xc5, 0x7f, 0x5f, 0x67, 0x24, 0xa3, 0x8e, 0x30, 0x2a, + 0xf8, 0xfb, 0x00, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0x58, 0xff, 0x22, 0x7b, 0xce, 0x0e, 0xa6, 0x5f, + 0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x2d, 0x43, 0x4c, 0xf9, 0x79, 0x54, 0xdb, 0x1f, 0x1d, 0x10, + 0x97, 0x5a, 0xd3, 0x37, 0xb0, 0xa8, 0xea, 0xf6, 0x6f, 0x42, 0x2e, 0xbc, 0x61, 0xd1, 0x1a, 0x64, + 0x9f, 0xb5, 0xf1, 0x93, 0x06, 0xd6, 0x16, 0xe4, 0x1c, 0x86, 0x35, 0xcf, 0x24, 0x45, 0x29, 0xc3, + 0xe2, 0x8e, 0xd1, 0x32, 0x1e, 0x37, 0x70, 0x98, 0x9a, 0x09, 0x01, 0xea, 0x9a, 0x28, 0x69, 0xaa, + 0x81, 0xc8, 0x66, 0xf5, 0xfa, 0x37, 0xdf, 0xae, 0x2d, 0xfc, 0xec, 0xdb, 0xb5, 0x85, 0x9f, 0x7f, + 0xbb, 0x96, 0x78, 0x71, 0xb6, 0x96, 0xf8, 0xe6, 0x6c, 0x2d, 0xf1, 0xd3, 0xb3, 0xb5, 0xc4, 0xbf, + 0x9d, 0xad, 0x25, 0xf6, 0xb2, 0x22, 0x08, 0xfa, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x7d, + 0xfe, 0xa7, 0xa7, 0xa8, 0x26, 0x00, 0x00, } diff --git a/api/types.proto b/api/types.proto index 88d8f87434..15c49c4a29 100644 --- a/api/types.proto +++ b/api/types.proto @@ -694,6 +694,13 @@ message RaftConfig { uint32 election_tick = 5; } +message EncryptionConfig { + // AutoLockManagers specifies whether or not managers TLS keys and raft data + // should be encrypted at rest in such a way that they must be unlocked + // before the manager node starts up again. + bool auto_lock_managers = 1; +} + // Placement specifies task distribution constraints. message Placement { // constraints specifies a set of requirements a node should meet for a task. diff --git a/ca/certificates.go b/ca/certificates.go index 19022d58ef..dc5dbc8f1c 100644 --- a/ca/certificates.go +++ b/ca/certificates.go @@ -23,12 +23,12 @@ import ( "github.com/docker/distribution/digest" "github.com/docker/go-events" "github.com/docker/swarmkit/api" - "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/ioutils" "github.com/docker/swarmkit/remotes" "github.com/pkg/errors" "golang.org/x/net/context" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" ) @@ -122,8 +122,8 @@ func (rca *RootCA) CanSign() bool { // IssueAndSaveNewCertificates generates a new key-pair, signs it with the local root-ca, and returns a // tls certificate -func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org string) (*tls.Certificate, error) { - csr, key, err := GenerateAndWriteNewKey(paths) +func (rca *RootCA) IssueAndSaveNewCertificates(kw KeyWriter, cn, ou, org string) (*tls.Certificate, error) { + csr, key, err := GenerateNewCSR() if err != nil { return nil, errors.Wrap(err, "error when generating new node certs") } @@ -138,20 +138,13 @@ func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org stri return nil, errors.Wrap(err, "failed to sign node certificate") } - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) + // Create a valid TLSKeyPair out of the PEM encoded private key and certificate + tlsKeyPair, err := tls.X509KeyPair(certChain, key) if err != nil { return nil, err } - // Write the chain to disk - if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil { - return nil, err - } - - // Create a valid TLSKeyPair out of the PEM encoded private key and certificate - tlsKeyPair, err := tls.X509KeyPair(certChain, key) - if err != nil { + if err := kw.Write(certChain, key, nil); err != nil { return nil, err } @@ -160,11 +153,9 @@ func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org stri // RequestAndSaveNewCertificates gets new certificates issued, either by signing them locally if a signer is // available, or by requesting them from the remote server at remoteAddr. -func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths CertPaths, token string, remotes remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) { - // Create a new key/pair and CSR for the new manager - // Write the new CSR and the new key to a temporary location so we can survive crashes on rotation - tempPaths := genTempPaths(paths) - csr, key, err := GenerateAndWriteNewKey(tempPaths) +func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, kw KeyWriter, token string, r remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) { + // Create a new key/pair and CSR + csr, key, err := GenerateNewCSR() if err != nil { return nil, errors.Wrap(err, "error when generating new node certs") } @@ -174,7 +165,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert // responding properly (for example, it may have just been demoted). var signedCert []byte for i := 0; i != 5; i++ { - signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, remotes, transport, nodeInfo) + signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, r, transport, nodeInfo) if err == nil { break } @@ -184,7 +175,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert } // Доверяй, но проверяй. - // Before we overwrite our local certificate, let's make sure the server gave us one that is valid + // Before we overwrite our local key + certificate, let's make sure the server gave us one that is valid // Create an X509Cert so we can .Verify() certBlock, _ := pem.Decode(signedCert) if certBlock == nil { @@ -209,23 +200,60 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert return nil, err } - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) + var kekUpdate *KEKData + for i := 0; i < 5; i++ { + kekUpdate, err = rca.getKEKUpdate(ctx, X509Cert, tlsKeyPair, r) + if err == nil { + break + } + } if err != nil { return nil, err } - // Write the chain to disk - if err := ioutils.AtomicWriteFile(paths.Cert, signedCert, 0644); err != nil { + if err := kw.Write(signedCert, key, kekUpdate); err != nil { return nil, err } - // Move the new key to the final location - if err := os.Rename(tempPaths.Key, paths.Key); err != nil { - return nil, err + return &tlsKeyPair, nil +} + +func (rca *RootCA) getKEKUpdate(ctx context.Context, cert *x509.Certificate, keypair tls.Certificate, r remotes.Remotes) (*KEKData, error) { + var managerRole bool + for _, ou := range cert.Subject.OrganizationalUnit { + if ou == ManagerRole { + managerRole = true + break + } } - return &tlsKeyPair, nil + if managerRole { + mtlsCreds := credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rca.Pool, Certificates: []tls.Certificate{keypair}}) + conn, peer, err := getGRPCConnection(mtlsCreds, r) + if err != nil { + return nil, err + } + defer conn.Close() + + client := api.NewCAClient(conn) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + response, err := client.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{}) + if err != nil { + if grpc.Code(err) == codes.Unimplemented { // if the server does not support keks, return as if no encryption key was specified + return &KEKData{}, nil + } + + r.Observe(peer, -remotes.DefaultObservationWeight) + return nil, err + } + r.Observe(peer, remotes.DefaultObservationWeight) + return &KEKData{KEK: response.UnlockKey, Version: response.Version.Index}, nil + } + + // If this is a worker, set to never encrypt. We always want to set to the lock key to nil, + // in case this was a manager that was demoted to a worker. + return &KEKData{}, nil } // PrepareCSR creates a CFSSL Sign Request based on the given raw CSR and @@ -388,11 +416,9 @@ func ensureCertKeyMatch(cert *x509.Certificate, key crypto.PublicKey) error { // GetLocalRootCA validates if the contents of the file are a valid self-signed // CA certificate, and returns the PEM-encoded Certificate if so -func GetLocalRootCA(baseDir string) (RootCA, error) { - paths := NewConfigPaths(baseDir) - +func GetLocalRootCA(paths CertPaths) (RootCA, error) { // Check if we have a Certificate file - cert, err := ioutil.ReadFile(paths.RootCA.Cert) + cert, err := ioutil.ReadFile(paths.Cert) if err != nil { if os.IsNotExist(err) { err = ErrNoLocalRootCA @@ -401,7 +427,7 @@ func GetLocalRootCA(baseDir string) (RootCA, error) { return RootCA{}, err } - key, err := ioutil.ReadFile(paths.RootCA.Key) + key, err := ioutil.ReadFile(paths.Key) if err != nil { if !os.IsNotExist(err) { return RootCA{}, err @@ -414,24 +440,31 @@ func GetLocalRootCA(baseDir string) (RootCA, error) { return NewRootCA(cert, key, DefaultNodeCertExpiration) } -// GetRemoteCA returns the remote endpoint's CA certificate -func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) { - // This TLS Config is intentionally using InsecureSkipVerify. Either we're - // doing TOFU, in which case we don't validate the remote CA, or we're using - // a user supplied hash to check the integrity of the CA certificate. - insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) +func getGRPCConnection(creds credentials.TransportCredentials, r remotes.Remotes) (*grpc.ClientConn, api.Peer, error) { + peer, err := r.Select() + if err != nil { + return nil, api.Peer{}, err + } + opts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecureCreds), + grpc.WithTransportCredentials(creds), grpc.WithTimeout(5 * time.Second), grpc.WithBackoffMaxDelay(5 * time.Second), } - peer, err := r.Select() + conn, err := grpc.Dial(peer.Addr, opts...) if err != nil { - return RootCA{}, err + return nil, api.Peer{}, err } + return conn, peer, nil +} - conn, err := grpc.Dial(peer.Addr, opts...) +// GetRemoteCA returns the remote endpoint's CA certificate +func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) { + // This TLS Config is intentionally using InsecureSkipVerify. We use the + // digest instead to check the integrity of the CA certificate. + insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) + conn, peer, err := getGRPCConnection(insecureCreds, r) if err != nil { return RootCA{}, err } @@ -481,9 +514,9 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootC return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil } -// CreateAndWriteRootCA creates a Certificate authority for a new Swarm Cluster, potentially +// CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially // overwriting any existing CAs. -func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) { +func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) { // Create a simple CSR for the CA using the default CA validator and policy req := cfcsr.CertificateRequest{ CN: rootCN, @@ -497,99 +530,17 @@ func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) { return RootCA{}, err } - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) + rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration) if err != nil { return RootCA{}, err } - // Write the Private Key and Certificate to disk, using decent permissions - if err := ioutils.AtomicWriteFile(paths.Cert, cert, 0644); err != nil { - return RootCA{}, err - } - if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil { + // save the cert to disk + if err := saveRootCA(rootCA, paths); err != nil { return RootCA{}, err } - return NewRootCA(cert, key, DefaultNodeCertExpiration) -} - -// BootstrapCluster receives a directory and creates both new Root CA key material -// and a ManagerRole key/certificate pair to be used by the initial cluster manager -func BootstrapCluster(baseCertDir string) error { - paths := NewConfigPaths(baseCertDir) - - rootCA, err := CreateAndWriteRootCA(rootCN, paths.RootCA) - if err != nil { - return err - } - - nodeID := identity.NewID() - newOrg := identity.NewID() - _, err = GenerateAndSignNewTLSCert(rootCA, nodeID, ManagerRole, newOrg, paths.Node) - - return err -} - -// GenerateAndSignNewTLSCert creates a new keypair, signs the certificate using signer, -// and saves the certificate and key to disk. This method is used to bootstrap the first -// manager TLS certificates. -func GenerateAndSignNewTLSCert(rootCA RootCA, cn, ou, org string, paths CertPaths) (*tls.Certificate, error) { - // Generate and new keypair and CSR - csr, key, err := generateNewCSR() - if err != nil { - return nil, err - } - - // Obtain a signed Certificate - certChain, err := rootCA.ParseValidateAndSignCSR(csr, cn, ou, org) - if err != nil { - return nil, errors.Wrap(err, "failed to sign node certificate") - } - - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) - if err != nil { - return nil, err - } - - // Write both the chain and key to disk - if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil { - return nil, err - } - if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil { - return nil, err - } - - // Load a valid tls.Certificate from the chain and the key - serverCert, err := tls.X509KeyPair(certChain, key) - if err != nil { - return nil, err - } - - return &serverCert, nil -} - -// GenerateAndWriteNewKey generates a new pub/priv key pair, writes it to disk -// and returns the CSR and the private key material -func GenerateAndWriteNewKey(paths CertPaths) (csr, key []byte, err error) { - // Generate a new key pair - csr, key, err = generateNewCSR() - if err != nil { - return - } - - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Key), 0755) - if err != nil { - return - } - - if err = ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil { - return - } - - return + return rootCA, nil } // GetRemoteSignedCertificate submits a CSR to a remote CA server address, @@ -605,18 +556,7 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r creds = credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rootCAPool}) } - peer, err := r.Select() - if err != nil { - return nil, err - } - - opts := []grpc.DialOption{ - grpc.WithTransportCredentials(creds), - grpc.WithTimeout(5 * time.Second), - grpc.WithBackoffMaxDelay(5 * time.Second), - } - - conn, err := grpc.Dial(peer.Addr, opts...) + conn, peer, err := getGRPCConnection(creds, r) if err != nil { return nil, err } @@ -681,10 +621,10 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r } // readCertValidity returns the certificate issue and expiration time -func readCertValidity(paths CertPaths) (time.Time, time.Time, error) { +func readCertValidity(kr KeyReader) (time.Time, time.Time, error) { var zeroTime time.Time // Read the Cert - cert, err := ioutil.ReadFile(paths.Cert) + cert, _, err := kr.Read() if err != nil { return zeroTime, zeroTime, err } @@ -714,7 +654,8 @@ func saveRootCA(rootCA RootCA, paths CertPaths) error { return ioutils.AtomicWriteFile(paths.Cert, rootCA.Cert, 0644) } -func generateNewCSR() (csr, key []byte, err error) { +// GenerateNewCSR returns a newly generated key and CSR signed with said key +func GenerateNewCSR() (csr, key []byte, err error) { req := &cfcsr.CertificateRequest{ KeyRequest: cfcsr.NewBasicKeyRequest(), } diff --git a/ca/certificates_test.go b/ca/certificates_test.go index 861be61360..7a2e5a0ba9 100644 --- a/ca/certificates_test.go +++ b/ca/certificates_test.go @@ -23,6 +23,7 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/phayes/permbits" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/net/context" ) @@ -42,34 +43,33 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestCreateAndWriteRootCA(t *testing.T) { +func TestCreateRootCA(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) - _, err = ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + _, err = ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) perms, err := permbits.Stat(paths.RootCA.Cert) assert.NoError(t, err) assert.False(t, perms.GroupWrite()) assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.RootCA.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) + + _, err = permbits.Stat(paths.RootCA.Key) + assert.True(t, os.IsNotExist(err)) } -func TestCreateAndWriteRootCAExpiry(t *testing.T) { +func TestCreateRootCAExpiry(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) // Convert the certificate into an object to create a RootCA @@ -89,26 +89,26 @@ func TestGetLocalRootCA(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) // First, try to load the local Root CA with the certificate missing. - _, err = ca.GetLocalRootCA(tempBaseDir) + _, err = ca.GetLocalRootCA(paths.RootCA) assert.Equal(t, ca.ErrNoLocalRootCA, err) // Create the local Root CA to ensure that we can reload it correctly. - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.True(t, rootCA.CanSign()) assert.NoError(t, err) - rootCA2, err := ca.GetLocalRootCA(tempBaseDir) + // No private key here + rootCA2, err := ca.GetLocalRootCA(paths.RootCA) assert.NoError(t, err) - assert.True(t, rootCA2.CanSign()) - assert.Equal(t, rootCA, rootCA2) + assert.Equal(t, rootCA.Cert, rootCA2.Cert) + assert.False(t, rootCA2.CanSign()) - // Try again, this time without a private key. - assert.NoError(t, os.Remove(paths.RootCA.Key)) - - rootCA3, err := ca.GetLocalRootCA(tempBaseDir) + // write private key and assert we can load it and sign + assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, rootCA.Key, os.FileMode(0600))) + rootCA3, err := ca.GetLocalRootCA(paths.RootCA) assert.NoError(t, err) - assert.False(t, rootCA3.CanSign()) assert.Equal(t, rootCA.Cert, rootCA3.Cert) + assert.True(t, rootCA3.CanSign()) // Try with a private key that does not match the CA cert public key. privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) @@ -121,62 +121,43 @@ func TestGetLocalRootCA(t *testing.T) { }) assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, privKeyPem, os.FileMode(0600))) - _, err = ca.GetLocalRootCA(tempBaseDir) + _, err = ca.GetLocalRootCA(paths.RootCA) assert.EqualError(t, err, "certificate key mismatch") } -func TestGenerateAndSignNewTLSCert(t *testing.T) { +func TestGetLocalRootCAInvalidCert(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) - assert.NoError(t, err) - - _, err = ca.GenerateAndSignNewTLSCert(rootCA, "CN", "OU", "ORG", paths.Node) - assert.NoError(t, err) + // Write some garbage to the CA cert + require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, []byte(`-----BEGIN CERTIFICATE-----\n +some random garbage\n +-----END CERTIFICATE-----`), 0644)) - perms, err := permbits.Stat(paths.Node.Cert) - assert.NoError(t, err) - assert.False(t, perms.GroupWrite()) - assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.Node.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) - - certBytes, err := ioutil.ReadFile(paths.Node.Cert) - assert.NoError(t, err) - certs, err := helpers.ParseCertificatesPEM(certBytes) - assert.NoError(t, err) - assert.Len(t, certs, 2) - assert.Equal(t, "CN", certs[0].Subject.CommonName) - assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0]) - assert.Equal(t, "ORG", certs[0].Subject.Organization[0]) - assert.Equal(t, "rootCN", certs[1].Subject.CommonName) + _, err = ca.GetLocalRootCA(paths.RootCA) + require.Error(t, err) } -func TestGenerateAndWriteNewKey(t *testing.T) { +func TestGetLocalRootCAInvalidKey(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) + // Create the local Root CA to ensure that we can reload it correctly. + _, err = ca.CreateRootCA("rootCN", paths.RootCA) + require.NoError(t, err) - csr, key, err := ca.GenerateAndWriteNewKey(paths.Node) - assert.NoError(t, err) - assert.NotNil(t, csr) - assert.NotNil(t, key) - - perms, err := permbits.Stat(paths.Node.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) + // Write some garbage to the root key - this will cause the loading to fail + require.NoError(t, ioutil.WriteFile(paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n +some random garbage\n +-----END EC PRIVATE KEY-----`), 0600)) - _, err = helpers.ParseCSRPEM(csr) - assert.NoError(t, err) + _, err = ca.GetLocalRootCA(paths.RootCA) + require.Error(t, err) } func TestEncryptECPrivateKey(t *testing.T) { @@ -184,9 +165,7 @@ func TestEncryptECPrivateKey(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) - paths := ca.NewConfigPaths(tempBaseDir) - - _, key, err := ca.GenerateAndWriteNewKey(paths.Node) + _, key, err := ca.GenerateNewCSR() assert.NoError(t, err) encryptedKey, err := ca.EncryptECPrivateKey(key, "passphrase") assert.NoError(t, err) @@ -204,10 +183,10 @@ func TestParseValidateAndSignCSR(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) - csr, _, err := ca.GenerateAndWriteNewKey(paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) signedCert, err := rootCA.ParseValidateAndSignCSR(csr, "CN", "OU", "ORG") @@ -232,7 +211,7 @@ func TestParseValidateAndSignMaliciousCSR(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) req := &cfcsr.CertificateRequest{ @@ -308,7 +287,7 @@ func TestRequestAndSaveNewCertificates(t *testing.T) { info := make(chan api.IssueNodeCertificateResponse, 1) // Copy the current RootCA without the signer rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool} - cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.Paths.Node, tc.WorkerToken, tc.Remotes, nil, info) + cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.ManagerToken, tc.Remotes, nil, info) assert.NoError(t, err) assert.NotNil(t, cert) perms, err := permbits.Stat(tc.Paths.Node.Cert) @@ -316,6 +295,51 @@ func TestRequestAndSaveNewCertificates(t *testing.T) { assert.False(t, perms.GroupWrite()) assert.False(t, perms.OtherWrite()) assert.NotEmpty(t, <-info) + + // there was no encryption config in the remote, so the key should be unencrypted + unencryptedKeyReader := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + _, _, err = unencryptedKeyReader.Read() + require.NoError(t, err) + + // the worker token is also unencrypted + cert, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.WorkerToken, tc.Remotes, nil, info) + assert.NoError(t, err) + assert.NotNil(t, cert) + assert.NotEmpty(t, <-info) + _, _, err = unencryptedKeyReader.Read() + require.NoError(t, err) + + // If there is a different kek in the remote store, when TLS certs are renewed the new key will + // be encrypted with that kek + assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error { + cluster := store.GetCluster(tx, tc.Organization) + cluster.Spec.EncryptionConfig.AutoLockManagers = true + cluster.UnlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: []byte("kek!"), + }} + return store.UpdateCluster(tx, cluster) + })) + assert.NoError(t, os.RemoveAll(tc.Paths.Node.Cert)) + assert.NoError(t, os.RemoveAll(tc.Paths.Node.Key)) + + _, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.ManagerToken, tc.Remotes, nil, info) + assert.NoError(t, err) + assert.NotEmpty(t, <-info) + + // key can no longer be read without a kek + _, _, err = unencryptedKeyReader.Read() + require.Error(t, err) + + _, _, err = ca.NewKeyReadWriter(tc.Paths.Node, []byte("kek!"), nil).Read() + require.NoError(t, err) + + // if it's a worker though, the key is always unencrypted, even though the manager key is encrypted + _, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.WorkerToken, tc.Remotes, nil, info) + assert.NoError(t, err) + assert.NotEmpty(t, <-info) + _, _, err = unencryptedKeyReader.Read() + require.NoError(t, err) } func TestIssueAndSaveNewCertificates(t *testing.T) { @@ -323,7 +347,7 @@ func TestIssueAndSaveNewCertificates(t *testing.T) { defer tc.Stop() // Test the creation of a manager certificate - cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.ManagerRole, tc.Organization) + cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.ManagerRole, tc.Organization) assert.NoError(t, err) assert.NotNil(t, cert) perms, err := permbits.Stat(tc.Paths.Node.Cert) @@ -345,7 +369,7 @@ func TestIssueAndSaveNewCertificates(t *testing.T) { assert.Contains(t, certs[0].DNSNames, "swarm-manager") // Test the creation of a worker node cert - cert, err = tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.WorkerRole, tc.Organization) + cert, err = tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.WorkerRole, tc.Organization) assert.NoError(t, err) assert.NotNil(t, cert) perms, err = permbits.Stat(tc.Paths.Node.Cert) @@ -372,7 +396,7 @@ func TestGetRemoteSignedCertificate(t *testing.T) { defer tc.Stop() // Create a new CSR to be signed - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) certs, err := ca.GetRemoteSignedCertificate(context.Background(), csr, tc.ManagerToken, tc.RootCA.Pool, tc.Remotes, nil, nil) @@ -404,7 +428,7 @@ func TestGetRemoteSignedCertificateNodeInfo(t *testing.T) { defer tc.Stop() // Create a new CSR to be signed - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) info := make(chan api.IssueNodeCertificateResponse, 1) @@ -421,7 +445,7 @@ func TestGetRemoteSignedCertificateWithPending(t *testing.T) { defer tc.Stop() // Create a new CSR to be signed - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) updates, cancel := state.Watch(tc.MemoryStore.WatchQueue(), state.EventCreateNode{}) @@ -448,35 +472,6 @@ func TestGetRemoteSignedCertificateWithPending(t *testing.T) { assert.NoError(t, <-completed) } -func TestBootstrapCluster(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") - assert.NoError(t, err) - defer os.RemoveAll(tempBaseDir) - - paths := ca.NewConfigPaths(tempBaseDir) - - err = ca.BootstrapCluster(tempBaseDir) - assert.NoError(t, err) - - perms, err := permbits.Stat(paths.RootCA.Cert) - assert.NoError(t, err) - assert.False(t, perms.GroupWrite()) - assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.RootCA.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) - - perms, err = permbits.Stat(paths.Node.Cert) - assert.NoError(t, err) - assert.False(t, perms.GroupWrite()) - assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.Node.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) -} - func TestNewRootCA(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) @@ -484,7 +479,7 @@ func TestNewRootCA(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, ca.DefaultNodeCertExpiration) @@ -499,12 +494,12 @@ func TestNewRootCABundle(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - // Write one root CA to disk, keep the bytes - secondRootCA, err := ca.CreateAndWriteRootCA("rootCN2", paths.RootCA) + // make one rootCA + secondRootCA, err := ca.CreateRootCA("rootCN2", paths.RootCA) assert.NoError(t, err) - // Overwrite the first root CA on disk - firstRootCA, err := ca.CreateAndWriteRootCA("rootCN1", paths.RootCA) + // make a second root CA + firstRootCA, err := ca.CreateRootCA("rootCN1", paths.RootCA) assert.NoError(t, err) // Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle @@ -517,14 +512,9 @@ func TestNewRootCABundle(t *testing.T) { assert.Equal(t, bundle, newRootCA.Cert) assert.Equal(t, 2, len(newRootCA.Pool.Subjects())) - // Now load the bundle from disk - diskRootCA, err := ca.GetLocalRootCA(tempBaseDir) - assert.NoError(t, err) - assert.Equal(t, bundle, diskRootCA.Cert) - assert.Equal(t, 2, len(diskRootCA.Pool.Subjects())) - - // If I use GenerateAndSignNewTLSCert to sign certs, I'll get the correct CA in the chain - _, err = ca.GenerateAndSignNewTLSCert(diskRootCA, "CN", "OU", "ORG", paths.Node) + // If I use newRootCA's IssueAndSaveNewCertificates to sign certs, I'll get the correct CA in the chain + kw := ca.NewKeyReadWriter(paths.Node, nil, nil) + _, err = newRootCA.IssueAndSaveNewCertificates(kw, "CN", "OU", "ORG") assert.NoError(t, err) certBytes, err := ioutil.ReadFile(paths.Node.Cert) @@ -546,14 +536,14 @@ func TestNewRootCANonDefaultExpiry(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, 1*time.Hour) assert.NoError(t, err) // Create and sign a new CSR - csr, _, err := ca.GenerateAndWriteNewKey(paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) cert, err := newRootCA.ParseValidateAndSignCSR(csr, "CN", ca.ManagerRole, "ORG") assert.NoError(t, err) @@ -588,7 +578,7 @@ func TestNewRootCAWithPassphrase(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) // Ensure that we're encrypting the Key bytes out of NewRoot if there diff --git a/ca/config.go b/ca/config.go index 872ab5e1cb..f6b5d150f0 100644 --- a/ca/config.go +++ b/ca/config.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io/ioutil" "math/big" "math/rand" "path/filepath" @@ -33,7 +32,8 @@ const ( nodeTLSKeyFilename = "swarm-node.key" nodeCSRFilename = "swarm-node.csr" - rootCN = "swarm-ca" + // DefaultRootCN represents the root CN that we should create roots CAs with by default + DefaultRootCN = "swarm-ca" // ManagerRole represents the Manager node type, and is used for authorization to endpoints ManagerRole = "swarm-manager" // WorkerRole represents the Worker node type, and is used for authorization to endpoints @@ -54,8 +54,9 @@ const ( type SecurityConfig struct { mu sync.Mutex - rootCA *RootCA - externalCA *ExternalCA + rootCA *RootCA + externalCA *ExternalCA + keyReadWriter *KeyReadWriter ServerTLSCreds *MutableTLSCreds ClientTLSCreds *MutableTLSCreds @@ -69,7 +70,7 @@ type CertificateUpdate struct { } // NewSecurityConfig initializes and returns a new SecurityConfig. -func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig { +func NewSecurityConfig(rootCA *RootCA, krw *KeyReadWriter, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig { // Make a new TLS config for the external CA client without a // ServerName value set. clientTLSConfig := clientTLSCreds.Config() @@ -82,6 +83,7 @@ func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTL return &SecurityConfig{ rootCA: rootCA, + keyReadWriter: krw, externalCA: NewExternalCA(rootCA, externalCATLSConfig), ClientTLSCreds: clientTLSCreds, ServerTLSCreds: serverTLSCreds, @@ -96,6 +98,16 @@ func (s *SecurityConfig) RootCA() *RootCA { return s.rootCA } +// KeyWriter returns the object that can write keys to disk +func (s *SecurityConfig) KeyWriter() KeyWriter { + return s.keyReadWriter +} + +// KeyReader returns the object that can read keys from disk +func (s *SecurityConfig) KeyReader() KeyReader { + return s.keyReadWriter +} + // UpdateRootCA replaces the root CA with a new root CA based on the specified // certificate, key, and the number of hours the certificates issue should last. func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration) error { @@ -181,70 +193,63 @@ func getCAHashFromToken(token string) (digest.Digest, error) { return digest.ParseDigest(fmt.Sprintf("sha256:%0[1]*s", 64, digestInt.Text(16))) } -// LoadOrCreateSecurityConfig encapsulates the security logic behind joining a cluster. -// Every node requires at least a set of TLS certificates with which to join the cluster with. -// In the case of a manager, these certificates will be used both for client and server credentials. -func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse) (*SecurityConfig, error) { - ctx = log.WithModule(ctx, "tls") - paths := NewConfigPaths(baseCertDir) - +// DownloadRootCA tries to retrieve a remote root CA and matches the digest against the provided token. +func DownloadRootCA(ctx context.Context, paths CertPaths, token string, r remotes.Remotes) (RootCA, error) { + var rootCA RootCA + // Get a digest for the optional CA hash string that we've been provided + // If we were provided a non-empty string, and it is an invalid hash, return + // otherwise, allow the invalid digest through. var ( - rootCA RootCA - serverTLSCreds, clientTLSCreds *MutableTLSCreds - err error + d digest.Digest + err error ) - - // Check if we already have a CA certificate on disk. We need a CA to have a valid SecurityConfig - rootCA, err = GetLocalRootCA(baseCertDir) - switch err { - case nil: - log.G(ctx).Debug("loaded CA certificate") - case ErrNoLocalRootCA: - log.G(ctx).WithError(err).Debugf("failed to load local CA certificate") - - // Get a digest for the optional CA hash string that we've been provided - // If we were provided a non-empty string, and it is an invalid hash, return - // otherwise, allow the invalid digest through. - var d digest.Digest - if token != "" { - d, err = getCAHashFromToken(token) - if err != nil { - return nil, err - } - } - - // Get the remote CA certificate, verify integrity with the - // hash provided. Retry up to 5 times, in case the manager we - // first try to contact is not responding properly (it may have - // just been demoted, for example). - - for i := 0; i != 5; i++ { - rootCA, err = GetRemoteCA(ctx, d, remotes) - if err == nil { - break - } - log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate") - } + if token != "" { + d, err = getCAHashFromToken(token) if err != nil { - return nil, err + return RootCA{}, err } - - // Save root CA certificate to disk - if err = saveRootCA(rootCA, paths.RootCA); err != nil { - return nil, err + } + // Get the remote CA certificate, verify integrity with the + // hash provided. Retry up to 5 times, in case the manager we + // first try to contact is not responding properly (it may have + // just been demoted, for example). + + for i := 0; i != 5; i++ { + rootCA, err = GetRemoteCA(ctx, d, r) + if err == nil { + break } + log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate") + } + if err != nil { + return RootCA{}, err + } - log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.RootCA.Cert) - default: - return nil, err + // Save root CA certificate to disk + if err = saveRootCA(rootCA, paths); err != nil { + return RootCA{}, err } + log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.Cert) + return rootCA, nil +} + +// LoadOrCreateSecurityConfig encapsulates the security logic behind joining a cluster. +// Every node requires at least a set of TLS certificates with which to join the cluster with. +// In the case of a manager, these certificates will be used both for client and server credentials. +func LoadOrCreateSecurityConfig(ctx context.Context, rootCA RootCA, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse, krw *KeyReadWriter) (*SecurityConfig, error) { + ctx = log.WithModule(ctx, "tls") + // At this point we've successfully loaded the CA details from disk, or // successfully downloaded them remotely. The next step is to try to // load our certificates. - clientTLSCreds, serverTLSCreds, err = LoadTLSCreds(rootCA, paths.Node) + clientTLSCreds, serverTLSCreds, err := LoadTLSCreds(rootCA, krw) if err != nil { - log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", paths.Node.Cert) + if _, ok := errors.Cause(err).(ErrInvalidKEK); ok { + return nil, err + } + + log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target()) var ( tlsKeyPair *tls.Certificate @@ -262,7 +267,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose NodeMembership: api.NodeMembershipAccepted, } } - tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(paths.Node, cn, proposedRole, org) + tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(krw, cn, proposedRole, org) if err != nil { log.G(ctx).WithFields(logrus.Fields{ "node.id": cn, @@ -278,7 +283,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose } else { // There was an error loading our Credentials, let's get a new certificate issued // Last argument is nil because at this point we don't have any valid TLS creds - tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, paths.Node, token, remotes, nil, nodeInfo) + tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, krw, token, remotes, nil, nodeInfo) if err != nil { log.G(ctx).WithError(err).Error("failed to request save new certificate") return nil, err @@ -299,7 +304,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose log.G(ctx).WithFields(logrus.Fields{ "node.id": clientTLSCreds.NodeID(), "node.role": clientTLSCreds.Role(), - }).Debugf("new node credentials generated: %s", paths.Node.Cert) + }).Debugf("new node credentials generated: %s", krw.Target()) } else { if nodeInfo != nil { nodeInfo <- api.IssueNodeCertificateResponse{ @@ -313,13 +318,66 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose }).Debug("loaded node credentials") } - return NewSecurityConfig(&rootCA, clientTLSCreds, serverTLSCreds), nil + return NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds), nil +} + +// RenewTLSConfigNow gets a new TLS cert and key, and updates the security config if provided. This is similar to +// RenewTLSConfig, except while that monitors for expiry, and periodically renews, this renews once and is blocking +func RenewTLSConfigNow(ctx context.Context, s *SecurityConfig, r remotes.Remotes) error { + ctx = log.WithModule(ctx, "tls") + log := log.G(ctx).WithFields(logrus.Fields{ + "node.id": s.ClientTLSCreds.NodeID(), + "node.role": s.ClientTLSCreds.Role(), + }) + + // Let's request new certs. Renewals don't require a token. + rootCA := s.RootCA() + tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx, + s.KeyWriter(), + "", + r, + s.ClientTLSCreds, + nil) + if err != nil { + log.WithError(err).Errorf("failed to renew the certificate") + return err + } + + clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole) + if err != nil { + log.WithError(err).Errorf("failed to create a new client config") + return err + } + serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool) + if err != nil { + log.WithError(err).Errorf("failed to create a new server config") + return err + } + + if err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig); err != nil { + log.WithError(err).Errorf("failed to update the client credentials") + return err + } + + // Update the external CA to use the new client TLS + // config using a copy without a serverName specified. + s.externalCA.UpdateTLSConfig(&tls.Config{ + Certificates: clientTLSConfig.Certificates, + RootCAs: clientTLSConfig.RootCAs, + MinVersion: tls.VersionTLS12, + }) + + if err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig); err != nil { + log.WithError(err).Errorf("failed to update the server TLS credentials") + return err + } + + return nil } // RenewTLSConfig will continuously monitor for the necessity of renewing the local certificates, either by // issuing them locally if key-material is available, or requesting them from a remote CA. -func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate { - paths := NewConfigPaths(baseCertDir) +func RenewTLSConfig(ctx context.Context, s *SecurityConfig, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate { updates := make(chan CertificateUpdate) go func() { @@ -337,10 +395,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, // Since the expiration of the certificate is managed remotely we should update our // retry timer on every iteration of this loop. // Retrieve the current certificate expiration information. - validFrom, validUntil, err := readCertValidity(paths.Node) + validFrom, validUntil, err := readCertValidity(s.KeyReader()) if err != nil { // We failed to read the expiration, let's stick with the starting default - log.Errorf("failed to read the expiration of the TLS certificate in: %s", paths.Node.Cert) + log.Errorf("failed to read the expiration of the TLS certificate in: %s", s.KeyReader().Target()) updates <- CertificateUpdate{Err: errors.New("failed to read certificate expiration")} } else { // If we have an expired certificate, we let's stick with the starting default in @@ -368,52 +426,12 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, return } - // Let's request new certs. Renewals don't require a token. - rootCA := s.RootCA() - tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx, - paths.Node, - "", - remotes, - s.ClientTLSCreds, - nil) - if err != nil { - log.WithError(err).Errorf("failed to renew the certificate") - updates <- CertificateUpdate{Err: err} - continue - } - - clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole) - if err != nil { - log.WithError(err).Errorf("failed to create a new client config") - updates <- CertificateUpdate{Err: err} - } - serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool) - if err != nil { - log.WithError(err).Errorf("failed to create a new server config") - updates <- CertificateUpdate{Err: err} - } - - err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig) - if err != nil { - log.WithError(err).Errorf("failed to update the client credentials") - updates <- CertificateUpdate{Err: err} - } - - // Update the external CA to use the new client TLS - // config using a copy without a serverName specified. - s.externalCA.UpdateTLSConfig(&tls.Config{ - Certificates: clientTLSConfig.Certificates, - RootCAs: clientTLSConfig.RootCAs, - MinVersion: tls.VersionTLS12, - }) - - err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig) - if err != nil { - log.WithError(err).Errorf("failed to update the server TLS credentials") + // ignore errors - it will just try again laster + if err := RenewTLSConfigNow(ctx, s, remotes); err != nil { updates <- CertificateUpdate{Err: err} + } else { + updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()} } - - updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()} } }() @@ -447,13 +465,9 @@ func calculateRandomExpiry(validFrom, validUntil time.Time) time.Duration { // LoadTLSCreds loads tls credentials from the specified path and verifies that // thay are valid for the RootCA. -func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLSCreds, error) { +func LoadTLSCreds(rootCA RootCA, kr KeyReader) (*MutableTLSCreds, *MutableTLSCreds, error) { // Read both the Cert and Key from disk - cert, err := ioutil.ReadFile(paths.Cert) - if err != nil { - return nil, nil, err - } - key, err := ioutil.ReadFile(paths.Key) + cert, key, err := kr.Read() if err != nil { return nil, nil, err } @@ -482,24 +496,9 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS // Now that we know this certificate is valid, create a TLS Certificate for our // credentials - var ( - keyPair tls.Certificate - newErr error - ) - keyPair, err = tls.X509KeyPair(cert, key) + keyPair, err := tls.X509KeyPair(cert, key) if err != nil { - // This current keypair isn't valid. It's possible we crashed before we - // overwrote the current key. Let's try loading it from disk. - tempPaths := genTempPaths(paths) - key, newErr = ioutil.ReadFile(tempPaths.Key) - if newErr != nil { - return nil, nil, err - } - - keyPair, newErr = tls.X509KeyPair(cert, key) - if newErr != nil { - return nil, nil, err - } + return nil, nil, err } // Load the Certificates as server credentials @@ -519,13 +518,6 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS return clientTLSCreds, serverTLSCreds, nil } -func genTempPaths(path CertPaths) CertPaths { - return CertPaths{ - Key: filepath.Join(filepath.Dir(path.Key), "."+filepath.Base(path.Key)), - Cert: filepath.Join(filepath.Dir(path.Cert), "."+filepath.Base(path.Cert)), - } -} - // NewServerTLSConfig returns a tls.Config configured for a TLS Server, given a tls.Certificate // and the PEM-encoded root CA Certificate func NewServerTLSConfig(cert *tls.Certificate, rootCAPool *x509.CertPool) (*tls.Config, error) { diff --git a/ca/config_test.go b/ca/config_test.go index 2c7d2dd206..2ab9df8450 100644 --- a/ca/config_test.go +++ b/ca/config_test.go @@ -16,8 +16,61 @@ import ( "github.com/docker/swarmkit/ioutils" "github.com/docker/swarmkit/manager/state/store" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestDownloadRootCASuccess(t *testing.T) { + tc := testutils.NewTestCA(t) + defer tc.Stop() + + // Remove the CA cert + os.RemoveAll(tc.Paths.RootCA.Cert) + + rootCA, err := ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, tc.WorkerToken, tc.Remotes) + require.NoError(t, err) + require.NotNil(t, rootCA.Pool) + require.NotNil(t, rootCA.Cert) + require.Nil(t, rootCA.Signer) + require.False(t, rootCA.CanSign()) + require.Equal(t, tc.RootCA.Cert, rootCA.Cert) + + // Remove the CA cert + os.RemoveAll(tc.Paths.RootCA.Cert) + + // downloading without a join token also succeeds + rootCA, err = ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, "", tc.Remotes) + require.NoError(t, err) + require.NotNil(t, rootCA.Pool) + require.NotNil(t, rootCA.Cert) + require.Nil(t, rootCA.Signer) + require.False(t, rootCA.CanSign()) + require.Equal(t, tc.RootCA.Cert, rootCA.Cert) +} + +func TestDownloadRootCAWrongCAHash(t *testing.T) { + tc := testutils.NewTestCA(t) + defer tc.Stop() + + // Remove the CA cert + os.RemoveAll(tc.Paths.RootCA.Cert) + + // invalid token + _, err := ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, "invalidtoken", tc.Remotes) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid join token") + + // invalid hash token + splitToken := strings.Split(tc.ManagerToken, "-") + splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" + replacementToken := strings.Join(splitToken, "-") + + os.RemoveAll(tc.Paths.RootCA.Cert) + + _, err = ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, replacementToken, tc.Remotes) + require.Error(t, err) + require.Contains(t, err.Error(), "remote CA does not match fingerprint.") +} + func TestLoadOrCreateSecurityConfigEmptyDir(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() @@ -25,15 +78,13 @@ func TestLoadOrCreateSecurityConfigEmptyDir(t *testing.T) { info := make(chan api.IssueNodeCertificateResponse, 1) // Remove all the contents from the temp dir and try again with a new node os.RemoveAll(tc.TempDir) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.Nil(t, nodeConfig.RootCA().Signer) - assert.False(t, nodeConfig.RootCA().CanSign()) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) assert.NotEmpty(t, <-info) } @@ -44,109 +95,27 @@ func TestLoadOrCreateSecurityConfigNoCerts(t *testing.T) { // Remove only the node certificates form the directory, and attest that we get // new certificates that are locally signed os.RemoveAll(tc.Paths.Node.Cert) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, tc.WorkerToken, ca.WorkerRole, tc.Remotes, nil) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, nil, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) - assert.True(t, nodeConfig.RootCA().CanSign()) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) info := make(chan api.IssueNodeCertificateResponse, 1) - // Remove only the node certificates form the directory, and attest that we get + // Remove only the node certificates form the directory, get a new rootCA, and attest that we get // new certificates that are issued by the remote CA - os.RemoveAll(tc.Paths.RootCA.Key) os.RemoveAll(tc.Paths.Node.Cert) - nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info) + rootCA, err := ca.GetLocalRootCA(tc.Paths.RootCA) assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.Nil(t, nodeConfig.RootCA().Signer) - assert.False(t, nodeConfig.RootCA().CanSign()) - assert.NotEmpty(t, <-info) -} - -func TestLoadOrCreateSecurityConfigWrongCAHash(t *testing.T) { - tc := testutils.NewTestCA(t) - defer tc.Stop() - - splitToken := strings.Split(tc.ManagerToken, "-") - splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" - replacementToken := strings.Join(splitToken, "-") - - info := make(chan api.IssueNodeCertificateResponse, 1) - // Remove only the node certificates form the directory, and attest that we get - // new certificates that are issued by the remote CA - os.RemoveAll(tc.Paths.RootCA.Key) - os.RemoveAll(tc.Paths.RootCA.Cert) - os.RemoveAll(tc.Paths.Node.Cert) - _, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, replacementToken, ca.WorkerRole, tc.Remotes, info) - assert.Error(t, err) - assert.Contains(t, err.Error(), "remote CA does not match fingerprint.") -} - -func TestLoadOrCreateSecurityConfigInvalidCACert(t *testing.T) { - tc := testutils.NewTestCA(t) - defer tc.Stop() - - // First load the current nodeConfig. We'll verify that after we corrupt - // the certificate, another subsequent call with get us new certs - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) + nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, rootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - // We have a valid signer because we bootstrapped with valid root key-material - assert.NotNil(t, nodeConfig.RootCA().Signer) - assert.True(t, nodeConfig.RootCA().CanSign()) - - // Write some garbage to the CA cert - ioutil.WriteFile(tc.Paths.RootCA.Cert, []byte(`-----BEGIN CERTIFICATE-----\n -some random garbage\n ------END CERTIFICATE-----`), 0644) - - // We should get an error when the CA cert is invalid. - _, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.Error(t, err) - - // Not having a local cert should cause us to fallback to using the - // picker to get a remote. - assert.Nil(t, os.Remove(tc.Paths.RootCA.Cert)) - - // Validate we got a new valid state - newNodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) - assert.True(t, nodeConfig.RootCA().CanSign()) - - // Ensure that we have the same certificate as before - assert.Equal(t, nodeConfig.RootCA().Cert, newNodeConfig.RootCA().Cert) -} - -func TestLoadOrCreateSecurityConfigInvalidCAKey(t *testing.T) { - tc := testutils.NewTestCA(t) - defer tc.Stop() - - // Write some garbage to the root key - ioutil.WriteFile(tc.Paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n -some random garbage\n ------END EC PRIVATE KEY-----`), 0644) - - // We should get an error when the local ca private key is invalid. - _, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.Error(t, err) + assert.Equal(t, rootCA, *nodeConfig.RootCA()) + assert.NotEmpty(t, <-info) } func TestLoadOrCreateSecurityConfigInvalidCert(t *testing.T) { @@ -158,14 +127,13 @@ func TestLoadOrCreateSecurityConfigInvalidCert(t *testing.T) { some random garbage\n -----END CERTIFICATE-----`), 0644) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, "", ca.WorkerRole, tc.Remotes, nil, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) } func TestLoadOrCreateSecurityConfigInvalidKey(t *testing.T) { @@ -177,41 +145,27 @@ func TestLoadOrCreateSecurityConfigInvalidKey(t *testing.T) { some random garbage\n -----END EC PRIVATE KEY-----`), 0644) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, "", ca.WorkerRole, tc.Remotes, nil, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) } -func TestLoadOrCreateSecurityConfigInvalidKeyWithValidTempKey(t *testing.T) { +func TestLoadOrCreateSecurityIncorrectPassphrase(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + paths := ca.NewConfigPaths(tc.TempDir) + _, err := tc.RootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(paths.Node, []byte("kek"), nil), + "nodeID", ca.WorkerRole, tc.Organization) + require.NoError(t, err) - // Write some garbage to the Key - ioutil.WriteFile(tc.Paths.Node.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n -some random garbage\n ------END EC PRIVATE KEY-----`), 0644) - nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, nil, nil) - assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + _, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, nil, + ca.NewKeyReadWriter(paths.Node, nil, nil)) + require.IsType(t, ca.ErrInvalidKEK{}, err) } func TestRenewTLSConfigWorker(t *testing.T) { @@ -240,7 +194,7 @@ func TestRenewTLSConfigWorker(t *testing.T) { }) // Create a new CSR and overwrite the key on disk - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, key, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issue a new certificate with the same details as the current config, but with 1 min expiration time @@ -253,8 +207,11 @@ func TestRenewTLSConfigWorker(t *testing.T) { err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644) assert.NoError(t, err) + err = ioutils.AtomicWriteFile(tc.Paths.Node.Key, key, 0600) + assert.NoError(t, err) + renew := make(chan struct{}) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) select { case <-time.After(10 * time.Second): assert.Fail(t, "TestRenewTLSConfig timed-out") @@ -291,7 +248,7 @@ func TestRenewTLSConfigManager(t *testing.T) { }) // Create a new CSR and overwrite the key on disk - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, key, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issue a new certificate with the same details as the current config, but with 1 min expiration time @@ -304,10 +261,13 @@ func TestRenewTLSConfigManager(t *testing.T) { err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644) assert.NoError(t, err) + err = ioutils.AtomicWriteFile(tc.Paths.Node.Key, key, 0600) + assert.NoError(t, err) + // Get a new nodeConfig with a TLS cert that has 1 minute to live renew := make(chan struct{}) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) select { case <-time.After(10 * time.Second): assert.Fail(t, "TestRenewTLSConfig timed-out") @@ -344,7 +304,7 @@ func TestRenewTLSConfigWithNoNode(t *testing.T) { }) // Create a new CSR and overwrite the key on disk - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, key, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issue a new certificate with the same details as the current config, but with 1 min expiration time @@ -357,6 +317,9 @@ func TestRenewTLSConfigWithNoNode(t *testing.T) { err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644) assert.NoError(t, err) + err = ioutils.AtomicWriteFile(tc.Paths.Node.Key, key, 0600) + assert.NoError(t, err) + // Delete the node from the backend store err = tc.MemoryStore.Update(func(tx store.Tx) error { node := store.GetNode(tx, nodeConfig.ClientTLSCreds.NodeID()) @@ -366,7 +329,7 @@ func TestRenewTLSConfigWithNoNode(t *testing.T) { assert.NoError(t, err) renew := make(chan struct{}) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) select { case <-time.After(10 * time.Second): assert.Fail(t, "TestRenewTLSConfig timed-out") @@ -390,7 +353,7 @@ func TestForceRenewTLSConfig(t *testing.T) { assert.NoError(t, err) renew := make(chan struct{}, 1) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) renew <- struct{}{} select { case <-time.After(10 * time.Second): diff --git a/ca/keyreadwriter.go b/ca/keyreadwriter.go new file mode 100644 index 0000000000..5cbb812671 --- /dev/null +++ b/ca/keyreadwriter.go @@ -0,0 +1,388 @@ +package ca + +import ( + "crypto/rand" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + + "crypto/tls" + + "github.com/docker/swarmkit/ioutils" + "github.com/pkg/errors" +) + +const ( + // keyPerms are the permissions used to write the TLS keys + keyPerms = 0600 + // certPerms are the permissions used to write TLS certificates + certPerms = 0644 + // versionHeader is the TLS PEM key header that contains the KEK version + versionHeader = "kek-version" +) + +// PEMKeyHeaders is something that needs to know about PEM headers when reading +// or writing TLS keys. +type PEMKeyHeaders interface { + // UnmarshalHeaders loads the headers map given the current KEK + UnmarshalHeaders(map[string]string, KEKData) (PEMKeyHeaders, error) + // MarshalHeaders returns a header map given the current KEK + MarshalHeaders(KEKData) (map[string]string, error) + // UpdateKEK may get a new PEMKeyHeaders if the KEK changes + UpdateKEK(KEKData, KEKData) PEMKeyHeaders +} + +// KeyReader reads a TLS cert and key from disk +type KeyReader interface { + Read() ([]byte, []byte, error) + Target() string +} + +// KeyWriter writes a TLS key and cert to disk +type KeyWriter interface { + Write([]byte, []byte, *KEKData) error + ViewAndUpdateHeaders(func(PEMKeyHeaders) (PEMKeyHeaders, error)) error + ViewAndRotateKEK(func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error + GetCurrentState() (PEMKeyHeaders, KEKData) + Target() string +} + +// KEKData provides an optional update to the kek when writing. The structure +// is needed so that we can tell the difference between "do not encrypt anymore" +// and there is "no update". +type KEKData struct { + KEK []byte + Version uint64 +} + +// ErrInvalidKEK means that we cannot decrypt the TLS key for some reason +type ErrInvalidKEK struct { + Wrapped error +} + +func (e ErrInvalidKEK) Error() string { + return e.Wrapped.Error() +} + +// KeyReadWriter is an object that knows how to read and write TLS keys and certs to disk, +// optionally encrypted and optionally updating PEM headers. +type KeyReadWriter struct { + mu sync.Mutex + kekData KEKData + paths CertPaths + headersObj PEMKeyHeaders +} + +// NewKeyReadWriter creates a new KeyReadWriter +func NewKeyReadWriter(paths CertPaths, kek []byte, headersObj PEMKeyHeaders) *KeyReadWriter { + return &KeyReadWriter{ + kekData: KEKData{KEK: kek}, + paths: paths, + headersObj: headersObj, + } +} + +// Migrate checks to see if a temporary key file exists. Older versions of +// swarmkit wrote temporary keys instead of temporary certificates, so +// migrate that temporary key if it exists. We want to write temporary certificates, +// instead of temporary keys, because we may need to periodically re-encrypt the +// keys and modify the headers, and it's easier to have a single canonical key +// location than two possible key locations. +func (k *KeyReadWriter) Migrate() error { + tmpPaths := k.genTempPaths() + keyBytes, err := ioutil.ReadFile(tmpPaths.Key) + if err != nil { + return nil // no key? no migration + } + + // it does exist - no need to decrypt, because previous versions of swarmkit + // which supported this temporary key did not support encrypting TLS keys + cert, err := ioutil.ReadFile(k.paths.Cert) + if err != nil { + return os.RemoveAll(tmpPaths.Key) // no cert? no migration + } + + // nope, this does not match the cert + if _, err = tls.X509KeyPair(cert, keyBytes); err != nil { + return os.RemoveAll(tmpPaths.Key) + } + + return os.Rename(tmpPaths.Key, k.paths.Key) +} + +// Read will read a TLS cert and key from the given paths +func (k *KeyReadWriter) Read() ([]byte, []byte, error) { + k.mu.Lock() + defer k.mu.Unlock() + keyBlock, err := k.readKey() + if err != nil { + return nil, nil, err + } + + if version, ok := keyBlock.Headers[versionHeader]; ok { + if versionInt, err := strconv.ParseUint(version, 10, 64); err == nil { + k.kekData.Version = versionInt + } + } + delete(keyBlock.Headers, versionHeader) + + if k.headersObj != nil { + newHeaders, err := k.headersObj.UnmarshalHeaders(keyBlock.Headers, k.kekData) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to read TLS key headers") + } + k.headersObj = newHeaders + } + + keyBytes := pem.EncodeToMemory(keyBlock) + cert, err := ioutil.ReadFile(k.paths.Cert) + // The cert is written to a temporary file first, then the key, and then + // the cert gets renamed - so, if interrupted, it's possible to end up with + // a cert that only exists in the temporary location. + switch { + case err == nil: + _, err = tls.X509KeyPair(cert, keyBytes) + case os.IsNotExist(err): //continue to try temp location + break + default: + return nil, nil, err + } + + // either the cert doesn't exist, or it doesn't match the key - try the temp file, if it exists + if err != nil { + var tempErr error + tmpPaths := k.genTempPaths() + cert, tempErr = ioutil.ReadFile(tmpPaths.Cert) + if tempErr != nil { + return nil, nil, err // return the original error + } + if _, tempErr := tls.X509KeyPair(cert, keyBytes); tempErr != nil { + os.RemoveAll(tmpPaths.Cert) // nope, it doesn't match either - remove and return the original error + return nil, nil, err + } + os.Rename(tmpPaths.Cert, k.paths.Cert) // try to move the temp cert back to the regular location + + } + + return cert, keyBytes, nil +} + +// ViewAndRotateKEK re-encrypts the key with a new KEK +func (k *KeyReadWriter) ViewAndRotateKEK(cb func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error { + k.mu.Lock() + defer k.mu.Unlock() + + updatedKEK, updatedHeaderObj, err := cb(k.kekData, k.headersObj) + if err != nil { + return err + } + + keyBlock, err := k.readKey() + if err != nil { + return err + } + + if err := k.writeKey(keyBlock, updatedKEK, updatedHeaderObj); err != nil { + return err + } + return nil +} + +// ViewAndUpdateHeaders updates the header manager, and updates any headers on the existing key +func (k *KeyReadWriter) ViewAndUpdateHeaders(cb func(PEMKeyHeaders) (PEMKeyHeaders, error)) error { + k.mu.Lock() + defer k.mu.Unlock() + + pkh, err := cb(k.headersObj) + if err != nil { + return err + } + + keyBlock, err := k.readKeyblock() + if err != nil { + return err + } + + headers := make(map[string]string) + if pkh != nil { + var err error + headers, err = pkh.MarshalHeaders(k.kekData) + if err != nil { + return err + } + } + // we WANT any original encryption headers + for key, value := range keyBlock.Headers { + normalizedKey := strings.TrimSpace(strings.ToLower(key)) + if normalizedKey == "proc-type" || normalizedKey == "dek-info" { + headers[key] = value + } + } + headers[versionHeader] = strconv.FormatUint(k.kekData.Version, 10) + keyBlock.Headers = headers + + if err = ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil { + return err + } + k.headersObj = pkh + return nil +} + +// GetCurrentState returns the current KEK data, including version +func (k *KeyReadWriter) GetCurrentState() (PEMKeyHeaders, KEKData) { + k.mu.Lock() + defer k.mu.Unlock() + return k.headersObj, k.kekData +} + +// Write attempts write a cert and key to text. This can also optionally update +// the KEK while writing, if an updated KEK is provided. If the pointer to the +// update KEK is nil, then we don't update. If the updated KEK itself is nil, +// then we update the KEK to be nil (data should be unencrypted). +func (k *KeyReadWriter) Write(certBytes, plaintextKeyBytes []byte, kekData *KEKData) error { + k.mu.Lock() + defer k.mu.Unlock() + + // current assumption is that the cert and key will be in the same directory + if err := os.MkdirAll(filepath.Dir(k.paths.Key), 0755); err != nil { + return err + } + + // Ensure that we will have a keypair on disk at all times by writing the cert to a + // temp path first. This is because we want to have only a single copy of the key + // for rotation and header modification. + tmpPaths := k.genTempPaths() + if err := ioutils.AtomicWriteFile(tmpPaths.Cert, certBytes, certPerms); err != nil { + return err + } + + keyBlock, _ := pem.Decode(plaintextKeyBytes) + if keyBlock == nil { + return errors.New("invalid PEM-encoded private key") + } + + if kekData == nil { + kekData = &k.kekData + } + pkh := k.headersObj + if k.headersObj != nil { + pkh = k.headersObj.UpdateKEK(k.kekData, *kekData) + } + + if err := k.writeKey(keyBlock, *kekData, pkh); err != nil { + return err + } + return os.Rename(tmpPaths.Cert, k.paths.Cert) +} + +func (k *KeyReadWriter) genTempPaths() CertPaths { + return CertPaths{ + Key: filepath.Join(filepath.Dir(k.paths.Key), "."+filepath.Base(k.paths.Key)), + Cert: filepath.Join(filepath.Dir(k.paths.Cert), "."+filepath.Base(k.paths.Cert)), + } +} + +// Target returns a string representation of this KeyReadWriter, namely where +// it is writing to +func (k *KeyReadWriter) Target() string { + return k.paths.Cert +} + +func (k *KeyReadWriter) readKeyblock() (*pem.Block, error) { + key, err := ioutil.ReadFile(k.paths.Key) + if err != nil { + return nil, err + } + + // Decode the PEM private key + keyBlock, _ := pem.Decode(key) + if keyBlock == nil { + return nil, errors.New("invalid PEM-encoded private key") + } + + return keyBlock, nil +} + +// readKey returns the decrypted key pem bytes, and enforces the KEK if applicable +// (writes it back with the correct encryption if it is not correctly encrypted) +func (k *KeyReadWriter) readKey() (*pem.Block, error) { + keyBlock, err := k.readKeyblock() + if err != nil { + return nil, err + } + + if !x509.IsEncryptedPEMBlock(keyBlock) { + return keyBlock, nil + } + + // If it's encrypted, we can't read without a passphrase (we're assuming + // empty passphrases iare invalid) + if k.kekData.KEK == nil { + return nil, ErrInvalidKEK{Wrapped: x509.IncorrectPasswordError} + } + + derBytes, err := x509.DecryptPEMBlock(keyBlock, k.kekData.KEK) + if err != nil { + return nil, ErrInvalidKEK{Wrapped: err} + } + // remove encryption PEM headers + headers := make(map[string]string) + mergePEMHeaders(headers, keyBlock.Headers) + + return &pem.Block{ + Type: keyBlock.Type, // the key type doesn't change + Bytes: derBytes, + Headers: headers, + }, nil +} + +// writeKey takes an unencrypted keyblock and, if the kek is not nil, encrypts it before +// writing it to disk. If the kek is nil, writes it to disk unencrypted. +func (k *KeyReadWriter) writeKey(keyBlock *pem.Block, kekData KEKData, pkh PEMKeyHeaders) error { + if kekData.KEK != nil { + encryptedPEMBlock, err := x509.EncryptPEMBlock(rand.Reader, + keyBlock.Type, + keyBlock.Bytes, + kekData.KEK, + x509.PEMCipherAES256) + if err != nil { + return err + } + if encryptedPEMBlock.Headers == nil { + return errors.New("unable to encrypt key - invalid PEM file produced") + } + keyBlock = encryptedPEMBlock + } + + if pkh != nil { + headers, err := pkh.MarshalHeaders(kekData) + if err != nil { + return err + } + mergePEMHeaders(keyBlock.Headers, headers) + } + keyBlock.Headers[versionHeader] = strconv.FormatUint(kekData.Version, 10) + + if err := ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil { + return err + } + k.kekData = kekData + k.headersObj = pkh + return nil +} + +// merges one set of PEM headers onto another, excepting for key encryption value +// "proc-type" and "dek-info" +func mergePEMHeaders(original, newSet map[string]string) { + for key, value := range newSet { + normalizedKey := strings.TrimSpace(strings.ToLower(key)) + if normalizedKey != "proc-type" && normalizedKey != "dek-info" { + original[key] = value + } + } +} diff --git a/ca/keyreadwriter_test.go b/ca/keyreadwriter_test.go new file mode 100644 index 0000000000..624adfef31 --- /dev/null +++ b/ca/keyreadwriter_test.go @@ -0,0 +1,414 @@ +package ca_test + +import ( + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/docker/swarmkit/ca" + "github.com/docker/swarmkit/ca/testutils" + "github.com/stretchr/testify/require" +) + +// can read and write tls keys that aren't encrypted, and that are encrypted. without +// a pem header manager, the headers are all preserved and not overwritten +func TestKeyReadWriter(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + expectedKey := key + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created + + checkCanReadWithKEK := func(kek []byte) *ca.KeyReadWriter { + k := ca.NewKeyReadWriter(path.Node, kek, nil) + readCert, readKey, err := k.Read() + require.NoError(t, err) + require.Equal(t, cert, readCert) + require.Equal(t, expectedKey, readKey, "Expected %s, Got %s", string(expectedKey), string(readKey)) + return k + } + + k := ca.NewKeyReadWriter(path.Node, nil, nil) + + // can't read things that don't exist + _, _, err = k.Read() + require.Error(t, err) + + // can write an unencrypted key with no updates + require.NoError(t, k.Write(cert, expectedKey, nil)) + + // can read unencrypted + k = checkCanReadWithKEK(nil) + _, kekData := k.GetCurrentState() + require.EqualValues(t, 0, kekData.Version) // the first version was 0 + + // write a key with headers to the key to make sure they're cleaned + keyBlock, _ := pem.Decode(expectedKey) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + expectedKey = pem.EncodeToMemory(keyBlock) + // write a version, but that's not what we'd expect back once we read + keyBlock.Headers["kek-version"] = "8" + require.NoError(t, ioutil.WriteFile(path.Node.Key, pem.EncodeToMemory(keyBlock), 0600)) + + // if a kek is provided, we can still read unencrypted keys, and read + // the provided version + k = checkCanReadWithKEK([]byte("original kek")) + _, kekData = k.GetCurrentState() + require.EqualValues(t, 8, kekData.Version) + + // we can update the kek and write at the same time + require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("new kek!"), Version: 3})) + + // the same kek can still read, and will continue to write with this key if + // no further kek updates are provided + _, _, err = k.Read() + require.NoError(t, err) + require.NoError(t, k.Write(cert, expectedKey, nil)) + + expectedKey = key + + // without the right kek, we can't read + k = ca.NewKeyReadWriter(path.Node, []byte("original kek"), nil) + _, _, err = k.Read() + require.Error(t, err) + + // same new key, just for sanity + k = checkCanReadWithKEK([]byte("new kek!")) + _, kekData = k.GetCurrentState() + require.EqualValues(t, 3, kekData.Version) + + // we can also change the kek back to nil, which means the key is unencrypted + require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: nil})) + k = checkCanReadWithKEK(nil) + _, kekData = k.GetCurrentState() + require.EqualValues(t, 0, kekData.Version) +} + +type testHeaders struct { + setHeaders func(map[string]string, ca.KEKData) (ca.PEMKeyHeaders, error) + newHeaders func(ca.KEKData) (map[string]string, error) +} + +func (p testHeaders) UnmarshalHeaders(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + if p.setHeaders != nil { + return p.setHeaders(h, k) + } + return nil, fmt.Errorf("set header error") +} + +func (p testHeaders) MarshalHeaders(k ca.KEKData) (map[string]string, error) { + if p.newHeaders != nil { + return p.newHeaders(k) + } + return nil, fmt.Errorf("update header error") +} + +func (p testHeaders) UpdateKEK(ca.KEKData, ca.KEKData) ca.PEMKeyHeaders { + return p +} + +// KeyReaderWriter makes a call to a get headers updater, if write is called, +// and set headers, if read is called. The KEK version header is always preserved +// no matter what. +func TestKeyReadWriterWithPemHeaderManager(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + // write a key with headers to the key to make sure it gets overwritten + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + key = pem.EncodeToMemory(keyBlock) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created + + // if if getting new headers fail, writing a key fails, and the key does not rotate + var count int + badKEKData := ca.KEKData{KEK: []byte("failed kek"), Version: 3} + k := ca.NewKeyReadWriter(path.Node, nil, testHeaders{newHeaders: func(k ca.KEKData) (map[string]string, error) { + if count == 0 { + count++ + require.Equal(t, badKEKData, k) + return nil, fmt.Errorf("fail") + } + require.Equal(t, ca.KEKData{}, k) + return nil, nil + }}) + // first write will fail + require.Error(t, k.Write(cert, key, &badKEKData)) + // the stored kek data will be not be updated because the write failed + _, kekData := k.GetCurrentState() + require.Equal(t, ca.KEKData{}, kekData) + // second write will succeed, using the original kek (nil) + require.NoError(t, k.Write(cert, key, nil)) + + var ( + headers map[string]string + kek ca.KEKData + ) + + // if setting headers fail, reading fails + k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(map[string]string, ca.KEKData) (ca.PEMKeyHeaders, error) { + return nil, fmt.Errorf("nope") + }}) + _, _, err = k.Read() + require.Error(t, err) + + k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + headers = h + kek = k + return testHeaders{}, nil + }}) + + _, _, err = k.Read() + require.NoError(t, err) + require.Equal(t, ca.KEKData{}, kek) + require.Equal(t, keyBlock.Headers, headers) + + // writing new headers is called with existing headers, and will write a key that has the headers + // returned by the header update function + k = ca.NewKeyReadWriter(path.Node, []byte("oldKek"), testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + require.Equal(t, []byte("newKEK"), kek.KEK) + return map[string]string{"updated": "headers"}, nil + }}) + require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("newKEK"), Version: 2})) + + // make sure headers were correctly set + k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + headers = h + kek = k + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) + require.Equal(t, ca.KEKData{KEK: []byte("newKEK"), Version: 2}, kek) + + _, kekData = k.GetCurrentState() + require.Equal(t, kek, kekData) + require.Equal(t, map[string]string{"updated": "headers"}, headers) +} + +func TestKeyReadWriterViewAndUpdateHeaders(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + + // write a key with headers to the key to make sure it gets passed when reading/writing headers + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + key = pem.EncodeToMemory(keyBlock) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644)) + require.NoError(t, ioutil.WriteFile(path.Node.Key, key, 0600)) + + // if the update headers callback function fails, updating headers fails + k := ca.NewKeyReadWriter(path.Node, nil, nil) + err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + require.Nil(t, h) + return nil, fmt.Errorf("nope") + }) + require.Error(t, err) + require.Equal(t, "nope", err.Error()) + + // updating headers succeed and is called with the latest kek data + err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + require.Nil(t, h) + return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + return map[string]string{"updated": "headers"}, nil + }}, nil + }) + require.NoError(t, err) + + k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + require.Equal(t, map[string]string{"updated": "headers"}, h) + require.Equal(t, ca.KEKData{}, k) + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) + + // we can also update headers on an encrypted key + k = ca.NewKeyReadWriter(path.Node, []byte("kek"), nil) + require.NoError(t, k.Write(cert, key, nil)) + + err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + require.Nil(t, h) + return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + require.Equal(t, ca.KEKData{KEK: []byte("kek")}, kek) + return map[string]string{"updated": "headers"}, nil + }}, nil + }) + require.NoError(t, err) + + k = ca.NewKeyReadWriter(path.Node, []byte("kek"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + require.Equal(t, map[string]string{"updated": "headers"}, h) + require.Equal(t, ca.KEKData{KEK: []byte("kek")}, k) + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) +} + +func TestKeyReadWriterViewAndRotateKEK(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + + // write a key with headers to the key to make sure it gets passed when reading/writing headers + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + key = pem.EncodeToMemory(keyBlock) + require.NoError(t, ca.NewKeyReadWriter(path.Node, nil, nil).Write(cert, key, nil)) + + // if if getting new kek and headers fail, rotating a KEK fails, and the kek does not rotate + k := ca.NewKeyReadWriter(path.Node, nil, nil) + require.Error(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) { + require.Equal(t, ca.KEKData{}, k) + require.Nil(t, h) + return ca.KEKData{}, nil, fmt.Errorf("Nope") + })) + + // writing new headers will write a key that has the headers returned by the header update function + k = ca.NewKeyReadWriter(path.Node, []byte("oldKEK"), nil) + require.NoError(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) { + require.Equal(t, ca.KEKData{KEK: []byte("oldKEK")}, k) + require.Nil(t, h) + return ca.KEKData{KEK: []byte("newKEK"), Version: uint64(2)}, + testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + require.Equal(t, []byte("newKEK"), kek.KEK) + return map[string]string{"updated": "headers"}, nil + }}, nil + })) + + // ensure the key has been re-encrypted and we can read it + k = ca.NewKeyReadWriter(path.Node, nil, nil) + _, _, err = k.Read() + require.Error(t, err) + + var headers map[string]string + + k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, _ ca.KEKData) (ca.PEMKeyHeaders, error) { + headers = h + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) + require.Equal(t, map[string]string{"updated": "headers"}, headers) +} + +// If we abort in the middle of writing the key and cert, such that only the key is written +// to the final location, when we read we can still read the cert from the temporary +// location. +func TestTwoPhaseReadWrite(t *testing.T) { + cert1, _, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + cert2, key2, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + krw := ca.NewKeyReadWriter(path.Node, nil, nil) + + // put a directory in the location where the cert goes, so we can't actually move + // the cert from the temporary location to the final location. + require.NoError(t, os.Mkdir(filepath.Join(path.Node.Cert), 0755)) + require.Error(t, krw.Write(cert2, key2, nil)) + + // the temp cert file should exist + tempCertPath := filepath.Join(filepath.Dir(path.Node.Cert), "."+filepath.Base(path.Node.Cert)) + readCert, err := ioutil.ReadFile(tempCertPath) + require.NoError(t, err) + require.Equal(t, cert2, readCert) + + // remove the directory, to simulate it failing to write the first time + os.RemoveAll(path.Node.Cert) + readCert, readKey, err := krw.Read() + require.NoError(t, err) + require.Equal(t, cert2, readCert) + require.Equal(t, key2, readKey) + // the cert should have been moved to its proper location + _, err = os.Stat(tempCertPath) + require.True(t, os.IsNotExist(err)) + + // If the cert in the proper location doesn't match the key, the temp location is checked + require.NoError(t, ioutil.WriteFile(tempCertPath, cert2, 0644)) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert1, 0644)) + readCert, readKey, err = krw.Read() + require.NoError(t, err) + require.Equal(t, cert2, readCert) + require.Equal(t, key2, readKey) + // the cert should have been moved to its proper location + _, err = os.Stat(tempCertPath) + require.True(t, os.IsNotExist(err)) + + // If the cert in the temp location also doesn't match, the failure matching the + // correctly-located cert is returned + require.NoError(t, os.Remove(path.Node.Cert)) + require.NoError(t, ioutil.WriteFile(tempCertPath, cert1, 0644)) // mismatching cert + _, _, err = krw.Read() + require.True(t, os.IsNotExist(err)) + // the cert should have been removed + _, err = os.Stat(tempCertPath) + require.True(t, os.IsNotExist(err)) +} + +func TestKeyReadWriterMigrate(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + + // if the key exists in an old location, migrate it from there. + tempKeyPath := filepath.Join(filepath.Dir(path.Node.Key), "."+filepath.Base(path.Node.Key)) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644)) + require.NoError(t, ioutil.WriteFile(tempKeyPath, key, 0600)) + + krw := ca.NewKeyReadWriter(path.Node, nil, nil) + require.NoError(t, krw.Migrate()) + _, err = os.Stat(tempKeyPath) + require.True(t, os.IsNotExist(err)) // it's been moved to the right place + _, _, err = krw.Read() + require.NoError(t, err) + + // migrate does not affect any existing files + dirList, err := ioutil.ReadDir(filepath.Dir(path.Node.Key)) + require.NoError(t, err) + require.NoError(t, krw.Migrate()) + dirList2, err := ioutil.ReadDir(filepath.Dir(path.Node.Key)) + require.NoError(t, err) + require.Equal(t, dirList, dirList2) + _, _, err = krw.Read() + require.NoError(t, err) +} diff --git a/ca/server.go b/ca/server.go index 5b27e73880..938f597058 100644 --- a/ca/server.go +++ b/ca/server.go @@ -69,6 +69,33 @@ func (s *Server) SetReconciliationRetryInterval(reconciliationRetryInterval time s.reconciliationRetryInterval = reconciliationRetryInterval } +// GetUnlockKey is responsible for returning the current unlock key used for encrypting TLS private keys and +// other at rest data. Access to this RPC call should only be allowed via mutual TLS from managers. +func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequest) (*api.GetUnlockKeyResponse, error) { + // This directly queries the store, rather than storing the unlock key and version on + // the `Server` object and updating it `updateCluster` is called, because we need this + // API to return the latest version of the key. Otherwise, there might be a slight delay + // between when the cluster gets updated, and when this function returns the latest key. + // This delay is currently unacceptable because this RPC call is the only way, after a + // cluster update, to get the actual value of the unlock key, and we don't want to return + // a cached value. + resp := api.GetUnlockKeyResponse{} + s.store.View(func(tx store.ReadTx) { + cluster := store.GetCluster(tx, s.securityConfig.ClientTLSCreds.Organization()) + resp.Version = cluster.Meta.Version + if cluster.Spec.EncryptionConfig.AutoLockManagers { + for _, encryptionKey := range cluster.UnlockKeys { + if encryptionKey.Subsystem == ManagerRole { + resp.UnlockKey = encryptionKey.Key + return + } + } + } + }) + + return &resp, nil +} + // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) { if request.NodeID == "" { diff --git a/ca/server_test.go b/ca/server_test.go index a4a01a6774..3465f7372c 100644 --- a/ca/server_test.go +++ b/ca/server_test.go @@ -1,6 +1,8 @@ package ca_test import ( + "bytes" + "fmt" "testing" "time" @@ -9,11 +11,15 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/ca/testutils" + raftutils "github.com/docker/swarmkit/manager/state/raft/testutils" "github.com/docker/swarmkit/manager/state/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var _ api.CAServer = &ca.Server{} +var _ api.NodeCAServer = &ca.Server{} + func TestGetRootCACertificate(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() @@ -43,7 +49,7 @@ func TestIssueNodeCertificate(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Token: tc.WorkerToken} @@ -65,7 +71,7 @@ func TestForceRotationIsNoop(t *testing.T) { defer tc.Stop() // Get a new Certificate issued - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Token: tc.WorkerToken} @@ -111,7 +117,7 @@ func TestIssueNodeCertificateBrokenCA(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) tc.ExternalSigningServer.Flake() @@ -157,7 +163,7 @@ func TestIssueNodeCertificateWorkerRenewal(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) role := api.NodeRoleWorker @@ -179,7 +185,7 @@ func TestIssueNodeCertificateManagerRenewal(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) assert.NotNil(t, csr) @@ -202,7 +208,7 @@ func TestIssueNodeCertificateWorkerFromDifferentOrgRenewal(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) // Since we're using a client that has a different Organization, this request will be treated @@ -216,7 +222,7 @@ func TestNodeCertificateRenewalsDoNotRequireToken(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) role := api.NodeRoleManager @@ -254,7 +260,7 @@ func TestNewNodeCertificateRequiresToken(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issuance fails if no secret is provided @@ -333,7 +339,7 @@ func TestNewNodeCertificateBadToken(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issuance fails if wrong secret is provided @@ -347,3 +353,51 @@ func TestNewNodeCertificateBadToken(t *testing.T) { _, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest) assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster") } + +func TestGetUnlockKey(t *testing.T) { + t.Parallel() + + tc := testutils.NewTestCA(t) + defer tc.Stop() + + var cluster *api.Cluster + tc.MemoryStore.View(func(tx store.ReadTx) { + clusters, err := store.FindClusters(tx, store.ByName(store.DefaultClusterName)) + require.NoError(t, err) + cluster = clusters[0] + }) + + resp, err := tc.CAClients[0].GetUnlockKey(context.Background(), &api.GetUnlockKeyRequest{}) + require.NoError(t, err) + require.Nil(t, resp.UnlockKey) + require.Equal(t, cluster.Meta.Version, resp.Version) + + // Update the unlock key + require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error { + cluster = store.GetCluster(tx, cluster.ID) + cluster.Spec.EncryptionConfig.AutoLockManagers = true + cluster.UnlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: []byte("secret"), + }} + return store.UpdateCluster(tx, cluster) + })) + + tc.MemoryStore.View(func(tx store.ReadTx) { + cluster = store.GetCluster(tx, cluster.ID) + }) + + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + resp, err = tc.CAClients[0].GetUnlockKey(context.Background(), &api.GetUnlockKeyRequest{}) + if err != nil { + return fmt.Errorf("get unlock key: %v", err) + } + if !bytes.Equal(resp.UnlockKey, []byte("secret")) { + return fmt.Errorf("secret hasn't rotated yet") + } + if cluster.Meta.Version.Index > resp.Version.Index { + return fmt.Errorf("hasn't updated to the right version yet") + } + return nil + }, 250*time.Millisecond)) +} diff --git a/ca/testutils/cautils.go b/ca/testutils/cautils.go index 089c7f6447..d249825c70 100644 --- a/ca/testutils/cautils.go +++ b/ca/testutils/cautils.go @@ -46,6 +46,7 @@ type TestCA struct { WorkerToken string ManagerToken string Remotes remotes.Remotes + KeyReadWriter *ca.KeyReadWriter } // Stop cleansup after TestCA @@ -65,27 +66,27 @@ func (tc *TestCA) Stop() { // NewNodeConfig returns security config for a new node, given a role func (tc *TestCA) NewNodeConfig(role string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, tc.Organization, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, tc.Organization, tc.TempDir, withNonSigningRoot) } // WriteNewNodeConfig returns security config for a new node, given a role // saving the generated key and certificates to disk func (tc *TestCA) WriteNewNodeConfig(role string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, tc.Organization, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, tc.Organization, tc.TempDir, withNonSigningRoot) } // NewNodeConfigOrg returns security config for a new node, given a role and an org func (tc *TestCA) NewNodeConfigOrg(role, org string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, org, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, org, tc.TempDir, withNonSigningRoot) } // WriteNewNodeConfigOrg returns security config for a new node, given a role and an org // saving the generated key and certificates to disk func (tc *TestCA) WriteNewNodeConfigOrg(role, org string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, org, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, org, tc.TempDir, withNonSigningRoot) } // External controls whether or not NewTestCA() will create a TestCA server @@ -94,7 +95,7 @@ var External bool // NewTestCA is a helper method that creates a TestCA and a bunch of default // connections and security configs. -func NewTestCA(t *testing.T) *TestCA { +func NewTestCA(t *testing.T, krwGenerators ...func(ca.CertPaths) *ca.KeyReadWriter) *TestCA { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) @@ -123,13 +124,18 @@ func NewTestCA(t *testing.T) *TestCA { } } - managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "", External) + krw := ca.NewKeyReadWriter(paths.Node, nil, nil) + if len(krwGenerators) > 0 { + krw = krwGenerators[0](paths.Node) + } + + managerConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, organization, "", External) assert.NoError(t, err) - managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "", External) + managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, "swarm-test-org-2", "", External) assert.NoError(t, err) - workerConfig, err := genSecurityConfig(s, rootCA, ca.WorkerRole, organization, "", External) + workerConfig, err := genSecurityConfig(s, rootCA, krw, ca.WorkerRole, organization, "", External) assert.NoError(t, err) l, err := net.Listen("tcp", "127.0.0.1:0") @@ -194,6 +200,7 @@ func NewTestCA(t *testing.T) *TestCA { WorkerToken: workerToken, ManagerToken: managerToken, Remotes: remotes, + KeyReadWriter: krw, } } @@ -224,7 +231,7 @@ func createNode(s *store.MemoryStore, nodeID, role string, csr, cert []byte) err return err } -func genSecurityConfig(s *store.MemoryStore, rootCA ca.RootCA, role, org, tmpDir string, nonSigningRoot bool) (*ca.SecurityConfig, error) { +func genSecurityConfig(s *store.MemoryStore, rootCA ca.RootCA, krw *ca.KeyReadWriter, role, org, tmpDir string, nonSigningRoot bool) (*ca.SecurityConfig, error) { req := &cfcsr.CertificateRequest{ KeyRequest: cfcsr.NewBasicKeyRequest(), } @@ -297,7 +304,7 @@ func genSecurityConfig(s *store.MemoryStore, rootCA ca.RootCA, role, org, tmpDir } } - return ca.NewSecurityConfig(&rootCA, nodeClientTLSCreds, nodeServerTLSCreds), nil + return ca.NewSecurityConfig(&rootCA, krw, nodeClientTLSCreds, nodeServerTLSCreds), nil } func createClusterObject(t *testing.T, s *store.MemoryStore, clusterID, workerToken, managerToken string, externalCAs ...*api.ExternalCA) { @@ -323,9 +330,8 @@ func createClusterObject(t *testing.T, s *store.MemoryStore, clusterID, workerTo })) } -// createAndWriteRootCA creates a Certificate authority for a new Swarm Cluster. -// We're copying ca.CreateAndWriteRootCA, so we can have smaller key-sizes for tests -func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duration) (ca.RootCA, error) { +// CreateRootCertAndKey returns a generated certificate and key for a root CA +func CreateRootCertAndKey(rootCN string) ([]byte, []byte, error) { // Create a simple CSR for the CA using the default CA validator and policy req := cfcsr.CertificateRequest{ CN: rootCN, @@ -335,6 +341,13 @@ func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duratio // Generate the CA and get the certificate and private key cert, _, key, err := initca.New(&req) + return cert, key, err +} + +// createAndWriteRootCA creates a Certificate authority for a new Swarm Cluster. +// We're copying ca.CreateRootCA, so we can have smaller key-sizes for tests +func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duration) (ca.RootCA, error) { + cert, key, err := CreateRootCertAndKey(rootCN) if err != nil { return ca.RootCA{}, err } diff --git a/ca/testutils/externalutils.go b/ca/testutils/externalutils.go index 6b91bfd60b..8110e3dd60 100644 --- a/ca/testutils/externalutils.go +++ b/ca/testutils/externalutils.go @@ -34,7 +34,7 @@ func NewExternalSigningServer(rootCA ca.RootCA, basedir string) (*ExternalSignin Cert: filepath.Join(basedir, "server.crt"), Key: filepath.Join(basedir, "server.key"), } - serverCert, err := ca.GenerateAndSignNewTLSCert(rootCA, serverCN, serverOU, "", serverPaths) + serverCert, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(serverPaths, nil, nil), serverCN, serverOU, "") if err != nil { return nil, errors.Wrap(err, "unable to get TLS server certificate") } diff --git a/ca/transport_test.go b/ca/transport_test.go index 2d9279ba09..6b0b5ba2eb 100644 --- a/ca/transport_test.go +++ b/ca/transport_test.go @@ -13,7 +13,7 @@ func TestNewMutableTLS(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.ManagerRole, tc.Organization) + cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.ManagerRole, tc.Organization) assert.NoError(t, err) tlsConfig, err := ca.NewServerTLSConfig(cert, tc.RootCA.Pool) @@ -28,7 +28,7 @@ func TestGetAndValidateCertificateSubject(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.ManagerRole, tc.Organization) + cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.ManagerRole, tc.Organization) assert.NoError(t, err) name, err := ca.GetAndValidateCertificateSubject([]tls.Certificate{*cert}) @@ -43,9 +43,9 @@ func TestLoadNewTLSConfig(t *testing.T) { defer tc.Stop() // Create two different certs and two different TLS configs - cert1, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN1", ca.ManagerRole, tc.Organization) + cert1, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN1", ca.ManagerRole, tc.Organization) assert.NoError(t, err) - cert2, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN2", ca.WorkerRole, tc.Organization) + cert2, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN2", ca.WorkerRole, tc.Organization) assert.NoError(t, err) tlsConfig1, err := ca.NewServerTLSConfig(cert1, tc.RootCA.Pool) assert.NoError(t, err) diff --git a/cmd/external-ca-example/main.go b/cmd/external-ca-example/main.go index 2037416404..f837422441 100644 --- a/cmd/external-ca-example/main.go +++ b/cmd/external-ca-example/main.go @@ -21,7 +21,7 @@ func main() { } // Initialize the Root CA. - rootCA, err := ca.CreateAndWriteRootCA("external-ca-example", rootPaths) + rootCA, err := ca.CreateRootCA("external-ca-example", rootPaths) if err != nil { logrus.Fatalf("unable to initialize Root CA: %s", err) } @@ -31,7 +31,9 @@ func main() { clusterID := identity.NewID() nodeID := identity.NewID() - if _, err := ca.GenerateAndSignNewTLSCert(rootCA, nodeID, ca.ManagerRole, clusterID, nodeConfigPaths.Node); err != nil { + + kw := ca.NewKeyReadWriter(nodeConfigPaths.Node, nil, nil) + if _, err := rootCA.IssueAndSaveNewCertificates(kw, nodeID, ca.ManagerRole, clusterID); err != nil { logrus.Fatalf("unable to create initial manager node credentials: %s", err) } diff --git a/cmd/swarmctl/cluster/cmd.go b/cmd/swarmctl/cluster/cmd.go index 5635796fe8..8905418dac 100644 --- a/cmd/swarmctl/cluster/cmd.go +++ b/cmd/swarmctl/cluster/cmd.go @@ -15,5 +15,6 @@ func init() { inspectCmd, listCmd, updateCmd, + unlockKeyCmd, ) } diff --git a/cmd/swarmctl/cluster/unlockkey.go b/cmd/swarmctl/cluster/unlockkey.go new file mode 100644 index 0000000000..808dd56ad9 --- /dev/null +++ b/cmd/swarmctl/cluster/unlockkey.go @@ -0,0 +1,50 @@ +package cluster + +import ( + "errors" + "fmt" + + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/cmd/swarmctl/common" + "github.com/docker/swarmkit/manager/encryption" + "github.com/spf13/cobra" +) + +// get the unlock key + +func displayUnlockKey(cmd *cobra.Command) error { + conn, err := common.DialConn(cmd) + if err != nil { + return err + } + defer conn.Close() + + resp, err := api.NewCAClient(conn).GetUnlockKey(common.Context(cmd), &api.GetUnlockKeyRequest{}) + if err != nil { + return err + } + + if len(resp.UnlockKey) == 0 { + fmt.Printf("Managers not auto-locked") + } + fmt.Printf("Managers auto-locked. Unlock key: %s\n", encryption.HumanReadableKey(resp.UnlockKey)) + return nil +} + +var ( + unlockKeyCmd = &cobra.Command{ + Use: "unlock-key ", + Short: "Get the unlock key for a cluster", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("cluster name missing") + } + + if len(args) > 1 { + return errors.New("unlock-key command takes exactly 1 argument") + } + + return displayUnlockKey(cmd) + }, + } +) diff --git a/cmd/swarmctl/cluster/update.go b/cmd/swarmctl/cluster/update.go index 9e23f03c6f..cb5d81921d 100644 --- a/cmd/swarmctl/cluster/update.go +++ b/cmd/swarmctl/cluster/update.go @@ -3,6 +3,7 @@ package cluster import ( "errors" "fmt" + "strings" "time" "github.com/docker/swarmkit/api" @@ -39,7 +40,7 @@ var ( flags := cmd.Flags() spec := &cluster.Spec - var rotation api.JoinTokenRotation + var rotation api.KeyRotation if flags.Changed("certexpiry") { cePeriod, err := flags.GetDuration("certexpiry") @@ -71,16 +72,28 @@ var ( if err != nil { return err } + rotateJoinToken = strings.ToLower(rotateJoinToken) switch rotateJoinToken { case "worker": - rotation.RotateWorkerToken = true + rotation.WorkerJoinToken = true case "manager": - rotation.RotateManagerToken = true + rotation.ManagerJoinToken = true default: - return errors.New("--rotate-join-token flag must be followed by worker or manager") + return errors.New("--rotate-join-token flag must be followed by 'worker' or 'manager'") } } + if flags.Changed("autolock") { + spec.EncryptionConfig.AutoLockManagers, err = flags.GetBool("autolock") + if err != nil { + return err + } + } + rotateUnlockKey, err := flags.GetBool("rotate-unlock-key") + if err != nil { + return err + } + rotation.ManagerUnlockKey = rotateUnlockKey driver, err := common.ParseLogDriverFlags(flags) if err != nil { @@ -98,6 +111,10 @@ var ( return err } fmt.Println(r.Cluster.ID) + + if rotation.ManagerUnlockKey { + return displayUnlockKey(cmd) + } return nil }, } @@ -112,4 +129,6 @@ func init() { updateCmd.Flags().String("log-driver", "", "Set default log driver for cluster") updateCmd.Flags().StringSlice("log-opt", nil, "Set options for default log driver") updateCmd.Flags().String("rotate-join-token", "", "Rotate join token for worker or manager") + updateCmd.Flags().Bool("rotate-unlock-key", false, "Rotate manager unlock key") + updateCmd.Flags().Bool("autolock", false, "Enable or disable manager autolocking (requiring an unlock key to start a stopped manager)") } diff --git a/cmd/swarmd/main.go b/cmd/swarmd/main.go index 035b78b043..7e09db0cea 100644 --- a/cmd/swarmd/main.go +++ b/cmd/swarmd/main.go @@ -14,6 +14,7 @@ import ( "github.com/docker/swarmkit/agent/exec/container" "github.com/docker/swarmkit/cli" "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/node" "github.com/docker/swarmkit/version" "github.com/spf13/cobra" @@ -117,6 +118,23 @@ var ( return err } + autolockManagers, err := cmd.Flags().GetBool("autolock") + if err != nil { + return err + } + + var unlockKey []byte + if cmd.Flags().Changed("unlock-key") { + unlockKeyString, err := cmd.Flags().GetString("unlock-key") + if err != nil { + return err + } + unlockKey, err = encryption.ParseHumanReadableKey(unlockKeyString) + if err != nil { + return err + } + } + // Create a cancellable context for our GRPC call ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -149,6 +167,8 @@ var ( Executor: executor, HeartbeatTick: hb, ElectionTick: election, + AutoLockManagers: autolockManagers, + UnlockKey: unlockKey, }) if err != nil { return err @@ -195,4 +215,6 @@ func init() { mainCmd.Flags().Uint32("heartbeat-tick", 1, "Defines the heartbeat interval (in seconds) for raft member health-check") mainCmd.Flags().Uint32("election-tick", 3, "Defines the amount of ticks (in seconds) needed without a Leader to trigger a new election") mainCmd.Flags().Var(&externalCAOpt, "external-ca", "Specifications of one or more certificate signing endpoints") + mainCmd.Flags().Bool("autolock", false, "Require an unlock key in order to start a manager once it's been stopped") + mainCmd.Flags().String("unlock-key", "", "Unlock this manager using this key") } diff --git a/manager/controlapi/cluster.go b/manager/controlapi/cluster.go index fd3cb3df99..f2015886e4 100644 --- a/manager/controlapi/cluster.go +++ b/manager/controlapi/cluster.go @@ -6,6 +6,7 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" @@ -107,12 +108,38 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe expireBlacklistedCerts(cluster) - if request.Rotation.RotateWorkerToken { + if request.Rotation.WorkerJoinToken { cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA) } - if request.Rotation.RotateManagerToken { + if request.Rotation.ManagerJoinToken { cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA) } + + var unlockKeys []*api.EncryptionKey + var managerKey *api.EncryptionKey + for _, eKey := range cluster.UnlockKeys { + if eKey.Subsystem == ca.ManagerRole { + if !cluster.Spec.EncryptionConfig.AutoLockManagers { + continue + } + managerKey = eKey + } + unlockKeys = append(unlockKeys, eKey) + } + + switch { + case !cluster.Spec.EncryptionConfig.AutoLockManagers: + break + case managerKey == nil: + unlockKeys = append(unlockKeys, &api.EncryptionKey{ + Subsystem: ca.ManagerRole, + Key: encryption.GenerateSecretKey(), + }) + case request.Rotation.ManagerUnlockKey: + managerKey.Key = encryption.GenerateSecretKey() + } + cluster.UnlockKeys = unlockKeys + return store.UpdateCluster(tx, cluster) }) if err != nil { diff --git a/manager/controlapi/cluster_test.go b/manager/controlapi/cluster_test.go index fe3c8f3906..85c433815e 100644 --- a/manager/controlapi/cluster_test.go +++ b/manager/controlapi/cluster_test.go @@ -1,6 +1,7 @@ package controlapi import ( + "fmt" "testing" "time" @@ -9,6 +10,7 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -25,11 +27,11 @@ func createClusterSpec(name string) *api.ClusterSpec { } } -func createCluster(t *testing.T, ts *testServer, id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster { +func createClusterObj(id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster { spec := createClusterSpec(name) spec.AcceptancePolicy = policy - cluster := &api.Cluster{ + return &api.Cluster{ ID: id, Spec: *spec, RootCA: api.RootCA{ @@ -42,6 +44,10 @@ func createCluster(t *testing.T, ts *testServer, id, name string, policy api.Acc }, }, } +} + +func createCluster(t *testing.T, ts *testServer, id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster { + cluster := createClusterObj(id, name, policy, rootCA) assert.NoError(t, ts.Store.Update(func(tx store.Tx) error { return store.CreateCluster(tx, cluster) })) @@ -244,8 +250,8 @@ func TestUpdateClusterRotateToken(t *testing.T) { ClusterID: cluster.ID, Spec: &cluster.Spec, ClusterVersion: &cluster.Meta.Version, - Rotation: api.JoinTokenRotation{ - RotateWorkerToken: true, + Rotation: api.KeyRotation{ + WorkerJoinToken: true, }, }) assert.NoError(t, err) @@ -266,8 +272,8 @@ func TestUpdateClusterRotateToken(t *testing.T) { ClusterID: cluster.ID, Spec: &cluster.Spec, ClusterVersion: &r.Clusters[0].Meta.Version, - Rotation: api.JoinTokenRotation{ - RotateManagerToken: true, + Rotation: api.KeyRotation{ + ManagerJoinToken: true, }, }) assert.NoError(t, err) @@ -288,9 +294,9 @@ func TestUpdateClusterRotateToken(t *testing.T) { ClusterID: cluster.ID, Spec: &cluster.Spec, ClusterVersion: &r.Clusters[0].Meta.Version, - Rotation: api.JoinTokenRotation{ - RotateWorkerToken: true, - RotateManagerToken: true, + Rotation: api.KeyRotation{ + WorkerJoinToken: true, + ManagerJoinToken: true, }, }) assert.NoError(t, err) @@ -306,6 +312,126 @@ func TestUpdateClusterRotateToken(t *testing.T) { assert.NotEqual(t, managerToken, r.Clusters[0].RootCA.JoinTokens.Manager) } +func TestUpdateClusterRotateUnlockKey(t *testing.T) { + ts := newTestServer(t) + defer ts.Stop() + // create a cluster with extra encryption keys, to make sure they exist + cluster := createClusterObj("id", "name", api.AcceptancePolicy{}, ts.Server.rootCA) + expected := make(map[string]*api.EncryptionKey) + for i := 1; i <= 2; i++ { + value := fmt.Sprintf("fake%d", i) + expected[value] = &api.EncryptionKey{Subsystem: value, Key: []byte(value)} + cluster.UnlockKeys = append(cluster.UnlockKeys, expected[value]) + } + require.NoError(t, ts.Store.Update(func(tx store.Tx) error { + return store.CreateCluster(tx, cluster) + })) + + // we have to get the key from the memory store, since the cluster returned by the API is redacted + getManagerKey := func() (managerKey *api.EncryptionKey) { + ts.Store.View(func(tx store.ReadTx) { + viewCluster := store.GetCluster(tx, cluster.ID) + // no matter whether there's a manager key or not, the other keys should not have been affected + foundKeys := make(map[string]*api.EncryptionKey) + for _, eKey := range viewCluster.UnlockKeys { + foundKeys[eKey.Subsystem] = eKey + } + for v, key := range expected { + foundKey, ok := foundKeys[v] + require.True(t, ok) + require.Equal(t, key, foundKey) + } + managerKey = foundKeys[ca.ManagerRole] + }) + return + } + + validateListResult := func(expectedLocked bool) api.Version { + r, err := ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{ + Filters: &api.ListClustersRequest_Filters{ + NamePrefixes: []string{"name"}, + }, + }) + + require.NoError(t, err) + require.Len(t, r.Clusters, 1) + require.Equal(t, expectedLocked, r.Clusters[0].Spec.EncryptionConfig.AutoLockManagers) + require.Nil(t, r.Clusters[0].UnlockKeys) // redacted + + return r.Clusters[0].Meta.Version + } + + // we start off with manager autolocking turned off + version := validateListResult(false) + require.Nil(t, getManagerKey()) + + // Rotate unlock key without turning auto-lock on - key should still be nil + _, err := ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: &cluster.Spec, + ClusterVersion: &version, + Rotation: api.KeyRotation{ + ManagerUnlockKey: true, + }, + }) + require.NoError(t, err) + version = validateListResult(false) + require.Nil(t, getManagerKey()) + + // Enable auto-lock only, no rotation boolean + spec := cluster.Spec.Copy() + spec.EncryptionConfig.AutoLockManagers = true + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: spec, + ClusterVersion: &version, + }) + require.NoError(t, err) + version = validateListResult(true) + managerKey := getManagerKey() + require.NotNil(t, managerKey) + + // Rotate the manager key + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: spec, + ClusterVersion: &version, + Rotation: api.KeyRotation{ + ManagerUnlockKey: true, + }, + }) + require.NoError(t, err) + version = validateListResult(true) + newManagerKey := getManagerKey() + require.NotNil(t, managerKey) + require.NotEqual(t, managerKey, newManagerKey) + managerKey = newManagerKey + + // Just update the cluster without modifying unlock keys + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: spec, + ClusterVersion: &version, + }) + require.NoError(t, err) + version = validateListResult(true) + newManagerKey = getManagerKey() + require.Equal(t, managerKey, newManagerKey) + + // Disable auto lock + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: &cluster.Spec, // set back to original spec + ClusterVersion: &version, + Rotation: api.KeyRotation{ + ManagerUnlockKey: true, // this will be ignored because we disable the auto-lock + }, + }) + require.NoError(t, err) + validateListResult(false) + require.Nil(t, getManagerKey()) +} + func TestListClusters(t *testing.T) { ts := newTestServer(t) defer ts.Stop() diff --git a/manager/deks.go b/manager/deks.go new file mode 100644 index 0000000000..4813a67d53 --- /dev/null +++ b/manager/deks.go @@ -0,0 +1,269 @@ +package manager + +import ( + "crypto/subtle" + "encoding/base64" + "fmt" + + "github.com/docker/swarmkit/ca" + "github.com/docker/swarmkit/manager/encryption" + "github.com/docker/swarmkit/manager/state/raft" +) + +const ( + // the raft DEK (data encryption key) is stored in the TLS key as a header + // these are the header values + pemHeaderRaftDEK = "raft-dek" + pemHeaderRaftPendingDEK = "raft-dek-pending" + pemHeaderRaftDEKNeedsRotation = "raft-dek-needs-rotation" +) + +// RaftDEKData contains all the data stored in TLS pem headers +type RaftDEKData struct { + raft.EncryptionKeys + NeedsRotation bool +} + +// UnmarshalHeaders loads the state of the DEK manager given the current TLS headers +func (r RaftDEKData) UnmarshalHeaders(headers map[string]string, kekData ca.KEKData) (ca.PEMKeyHeaders, error) { + var ( + currentDEK, pendingDEK []byte + err error + ) + + if currentDEKStr, ok := headers[pemHeaderRaftDEK]; ok { + currentDEK, err = decodePEMHeaderValue(currentDEKStr, kekData.KEK) + if err != nil { + return nil, err + } + } + if pendingDEKStr, ok := headers[pemHeaderRaftPendingDEK]; ok { + pendingDEK, err = decodePEMHeaderValue(pendingDEKStr, kekData.KEK) + if err != nil { + return nil, err + } + } + + if pendingDEK != nil && currentDEK == nil { + return nil, fmt.Errorf("there is a pending DEK, but no current DEK") + } + + _, ok := headers[pemHeaderRaftDEKNeedsRotation] + return RaftDEKData{ + NeedsRotation: ok, + EncryptionKeys: raft.EncryptionKeys{ + CurrentDEK: currentDEK, + PendingDEK: pendingDEK, + }, + }, nil +} + +// MarshalHeaders returns new headers given the current KEK +func (r RaftDEKData) MarshalHeaders(kekData ca.KEKData) (map[string]string, error) { + headers := make(map[string]string) + for headerKey, contents := range map[string][]byte{ + pemHeaderRaftDEK: r.CurrentDEK, + pemHeaderRaftPendingDEK: r.PendingDEK, + } { + if contents != nil { + dekStr, err := encodePEMHeaderValue(contents, kekData.KEK) + if err != nil { + return nil, err + } + headers[headerKey] = dekStr + } + } + + if r.NeedsRotation { + headers[pemHeaderRaftDEKNeedsRotation] = "true" + } + + // return a function that updates the dek data on write success + return headers, nil +} + +// UpdateKEK optionally sets NeedRotation to true if we go from unlocked to locked +func (r RaftDEKData) UpdateKEK(oldKEK, candidateKEK ca.KEKData) ca.PEMKeyHeaders { + if _, unlockedToLocked, err := compareKEKs(oldKEK, candidateKEK); err == nil && unlockedToLocked { + return RaftDEKData{ + EncryptionKeys: r.EncryptionKeys, + NeedsRotation: true, + } + } + return r +} + +// Returns whether the old KEK should be replaced with the new KEK, whether we went from +// unlocked to locked, and whether there was an error (the versions are the same, but the +// keks are different) +func compareKEKs(oldKEK, candidateKEK ca.KEKData) (bool, bool, error) { + keksEqual := subtle.ConstantTimeCompare(oldKEK.KEK, candidateKEK.KEK) == 1 + switch { + case oldKEK.Version == candidateKEK.Version && !keksEqual: + return false, false, fmt.Errorf("candidate KEK has the same version as the current KEK, but a different KEK value") + case oldKEK.Version >= candidateKEK.Version || keksEqual: + return false, false, nil + default: + return true, oldKEK.KEK == nil, nil + } +} + +// RaftDEKManager manages the raft DEK keys using TLS headers +type RaftDEKManager struct { + kw ca.KeyWriter + rotationCh chan struct{} +} + +var errNoUpdateNeeded = fmt.Errorf("don't need to rotate or update") + +// this error is returned if the KeyReadWriter's PEMKeyHeaders object is no longer a RaftDEKData object - +// this can happen if the node is no longer a manager, for example +var errNotUsingRaftDEKData = fmt.Errorf("RaftDEKManager can no longer store and manage TLS key headers") + +// NewRaftDEKManager returns a RaftDEKManager that uses the current key writer +// and header manager +func NewRaftDEKManager(kw ca.KeyWriter) (*RaftDEKManager, error) { + // If there is no current DEK, generate one and write it to disk + err := kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + dekData, ok := h.(RaftDEKData) + // it wasn't a raft DEK manager before - just replace it + if !ok || dekData.CurrentDEK == nil { + return RaftDEKData{ + EncryptionKeys: raft.EncryptionKeys{ + CurrentDEK: encryption.GenerateSecretKey(), + }, + }, nil + } + return nil, errNoUpdateNeeded + }) + if err != nil && err != errNoUpdateNeeded { + return nil, err + } + return &RaftDEKManager{ + kw: kw, + rotationCh: make(chan struct{}, 1), + }, nil +} + +// NeedsRotation returns a boolean about whether we should do a rotation +func (r *RaftDEKManager) NeedsRotation() bool { + h, _ := r.kw.GetCurrentState() + data, ok := h.(RaftDEKData) + if !ok { + return false + } + return data.NeedsRotation || data.EncryptionKeys.PendingDEK != nil +} + +// GetKeys returns the current set of DEKs. If NeedsRotation is true, and there +// is no existing PendingDEK, it will try to create one. If there are any errors +// doing so, just return the original. +func (r *RaftDEKManager) GetKeys() raft.EncryptionKeys { + var newKeys, originalKeys raft.EncryptionKeys + err := r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + data, ok := h.(RaftDEKData) + if !ok { + return nil, errNotUsingRaftDEKData + } + originalKeys = data.EncryptionKeys + if !data.NeedsRotation || data.PendingDEK != nil { + return nil, errNoUpdateNeeded + } + newKeys = raft.EncryptionKeys{ + CurrentDEK: data.CurrentDEK, + PendingDEK: encryption.GenerateSecretKey(), + } + return RaftDEKData{EncryptionKeys: newKeys}, nil + }) + if err != nil { + return originalKeys + } + return newKeys +} + +// RotationNotify the channel used to notify subscribers as to whether there +// should be a rotation done +func (r *RaftDEKManager) RotationNotify() chan struct{} { + return r.rotationCh +} + +// UpdateKeys will set the updated encryption keys in the headers. This finishes +// a rotation, and is expected to set the CurrentDEK to the previous PendingDEK. +func (r *RaftDEKManager) UpdateKeys(newKeys raft.EncryptionKeys) error { + return r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + data, ok := h.(RaftDEKData) + if !ok { + return nil, errNotUsingRaftDEKData + } + // If there is no current DEK, we are basically wiping out all DEKs (no header object) + if newKeys.CurrentDEK == nil { + return nil, nil + } + return RaftDEKData{ + EncryptionKeys: newKeys, + NeedsRotation: data.NeedsRotation, + }, nil + }) +} + +// MaybeUpdateKEK does a KEK rotation if one is required. Returns whether +// the kek was updated, whether it went from unlocked to locked, and any errors. +func (r *RaftDEKManager) MaybeUpdateKEK(candidateKEK ca.KEKData) (bool, bool, error) { + var updated, unlockedToLocked bool + err := r.kw.ViewAndRotateKEK(func(currentKEK ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) { + var err error + updated, unlockedToLocked, err = compareKEKs(currentKEK, candidateKEK) + if err == nil && !updated { // if we don't need to rotate the KEK, don't bother updating + err = errNoUpdateNeeded + } + if err != nil { + return ca.KEKData{}, nil, err + } + + data, ok := h.(RaftDEKData) + if !ok { + return ca.KEKData{}, nil, errNotUsingRaftDEKData + } + + if unlockedToLocked { + data.NeedsRotation = true + } + return candidateKEK, data, nil + }) + if err == errNoUpdateNeeded { + err = nil + } + + if err == nil && unlockedToLocked { + r.rotationCh <- struct{}{} + } + return updated, unlockedToLocked, err +} + +func decodePEMHeaderValue(headerValue string, kek []byte) ([]byte, error) { + var decrypter encryption.Decrypter = encryption.NoopCrypter + if kek != nil { + _, decrypter = encryption.Defaults(kek) + } + valueBytes, err := base64.StdEncoding.DecodeString(headerValue) + if err != nil { + return nil, err + } + result, err := encryption.Decrypt(valueBytes, decrypter) + if err != nil { + return nil, ca.ErrInvalidKEK{Wrapped: err} + } + return result, nil +} + +func encodePEMHeaderValue(headerValue []byte, kek []byte) (string, error) { + var encrypter encryption.Encrypter = encryption.NoopCrypter + if kek != nil { + encrypter, _ = encryption.Defaults(kek) + } + encrypted, err := encryption.Encrypt(headerValue, encrypter) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(encrypted), nil +} diff --git a/manager/deks_test.go b/manager/deks_test.go new file mode 100644 index 0000000000..55ed40debb --- /dev/null +++ b/manager/deks_test.go @@ -0,0 +1,464 @@ +package manager + +import ( + "encoding/base64" + "encoding/pem" + "io/ioutil" + "os" + "testing" + + "github.com/docker/swarmkit/ca" + cautils "github.com/docker/swarmkit/ca/testutils" + "github.com/docker/swarmkit/manager/state/raft" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +// Tests updating a kek on a raftDEK object. +func TestRaftDEKUpdateKEK(t *testing.T) { + startData := RaftDEKData{ + EncryptionKeys: raft.EncryptionKeys{CurrentDEK: []byte("first dek")}, + } + startKEK := ca.KEKData{} + + // because UpdateKEK returns a PEMKeyHeaders interface, we need to cast to check + // values + updateDEKAndCast := func(dekdata RaftDEKData, oldKEK ca.KEKData, newKEK ca.KEKData) RaftDEKData { + result := dekdata.UpdateKEK(oldKEK, newKEK) + raftDekObj, ok := result.(RaftDEKData) + require.True(t, ok) + return raftDekObj + } + + // nothing changes if we are updating a kek and they're both nil + result := updateDEKAndCast(startData, startKEK, ca.KEKData{Version: 2}) + require.Equal(t, result, startData) + + // when moving from unlocked to locked, a "needs rotation" header is generated but no + // pending header is generated + updatedKEK := ca.KEKData{KEK: []byte("something"), Version: 1} + result = updateDEKAndCast(startData, startKEK, updatedKEK) + require.NotEqual(t, startData, result) + require.True(t, result.NeedsRotation) + require.Equal(t, startData.CurrentDEK, result.CurrentDEK) + require.Nil(t, result.PendingDEK) + + // this is whether or not pending exists + startData.PendingDEK = []byte("pending") + result = updateDEKAndCast(startData, startKEK, updatedKEK) + require.NotEqual(t, startData, result) + require.True(t, result.NeedsRotation) + require.Equal(t, startData.CurrentDEK, result.CurrentDEK) + require.Equal(t, startData.PendingDEK, result.PendingDEK) + + // if we are going from locked to unlocked, nothing happens + result = updateDEKAndCast(startData, updatedKEK, startKEK) + require.Equal(t, startData, result) + require.False(t, result.NeedsRotation) + + // if we are going to locked to another locked, nothing happens + result = updateDEKAndCast(startData, updatedKEK, ca.KEKData{KEK: []byte("other"), Version: 4}) + require.Equal(t, startData, result) + require.False(t, result.NeedsRotation) +} + +func TestRaftDEKMarshalUnmarshal(t *testing.T) { + startData := RaftDEKData{ + EncryptionKeys: raft.EncryptionKeys{CurrentDEK: []byte("first dek")}, + } + kek := ca.KEKData{} + + headers, err := startData.MarshalHeaders(kek) + require.NoError(t, err) + require.Len(t, headers, 1) + + // can't unmarshal with the wrong kek + _, err = RaftDEKData{}.UnmarshalHeaders(headers, ca.KEKData{KEK: []byte("something")}) + require.Error(t, err) + + // we can unmarshal what was marshalled with the right kek + toData, err := RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.NoError(t, err) + require.Equal(t, startData, toData) + + // try the other headers as well + startData.PendingDEK = []byte("Hello") + headers, err = startData.MarshalHeaders(kek) + require.NoError(t, err) + require.Len(t, headers, 2) + + // we can unmarshal what was marshalled + toData, err = RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.NoError(t, err) + require.Equal(t, startData, toData) + + // try the other headers as well + startData.NeedsRotation = true + startData.PendingDEK = nil + headers, err = startData.MarshalHeaders(kek) + require.NoError(t, err) + require.Len(t, headers, 2) + + // we can unmarshal what was marshalled + toData, err = RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.NoError(t, err) + require.Equal(t, startData, toData) + + // If there is a pending header, but no current header, set will fail + headers = map[string]string{ + pemHeaderRaftPendingDEK: headers[pemHeaderRaftDEK], + } + _, err = RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.Error(t, err) + require.Contains(t, err.Error(), "pending DEK, but no current DEK") +} + +// NewRaftDEKManager creates a key if one doesn't exist +func TestNewRaftDEKManager(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-new-dek-manager-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + krw := ca.NewKeyReadWriter(paths.Node, nil, nil) + require.NoError(t, krw.Write(cert, key, nil)) + + keyBytes, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.NotContains(t, string(keyBytes), pemHeaderRaftDEK) // headers are not written + + dekManager, err := NewRaftDEKManager(krw) // this should create a new DEK and write it to the file + require.NoError(t, err) + + keyBytes, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Contains(t, string(keyBytes), pemHeaderRaftDEK) // header is written now + + keys := dekManager.GetKeys() + require.NotNil(t, keys.CurrentDEK) + require.Nil(t, keys.PendingDEK) + require.False(t, dekManager.NeedsRotation()) + + // If one exists, nothing is updated + dekManager, err = NewRaftDEKManager(krw) // this should create a new DEK and write it to the file + require.NoError(t, err) + + keyBytes2, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2) + + require.Equal(t, keys, dekManager.GetKeys()) + require.False(t, dekManager.NeedsRotation()) +} + +// NeedsRotate returns true if there is a PendingDEK or a NeedsRotation flag +func TestRaftDEKManagerNeedsRotateGetKeys(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-maybe-get-data-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + + // if there is no PendingDEK, and no NeedsRotation flag: NeedsRotation=false + keys := raft.EncryptionKeys{CurrentDEK: []byte("hello")} + dekManager, err := NewRaftDEKManager( + ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{EncryptionKeys: keys})) + require.NoError(t, err) + + require.False(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + + // if there is a PendingDEK, and no NeedsRotation flag: NeedsRotation=true + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello"), PendingDEK: []byte("another")} + dekManager, err = NewRaftDEKManager( + ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{EncryptionKeys: keys})) + require.NoError(t, err) + + require.True(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + + // if there is a PendingDEK, and a NeedsRotation flag: NeedsRotation=true + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello"), PendingDEK: []byte("another")} + dekManager, err = NewRaftDEKManager( + ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + })) + require.NoError(t, err) + + require.True(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + + // if there no PendingDEK, and a NeedsRotation flag: NeedsRotation=true and + // GetKeys attempts to create a pending key and write it to disk. However, writing + // will error (because there is no key on disk atm), and then the original keys will + // be returned. + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello")} + krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + }) + dekManager, err = NewRaftDEKManager(krw) + require.NoError(t, err) + + require.True(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + h, _ := krw.GetCurrentState() + dekData, ok := h.(RaftDEKData) + require.True(t, ok) + require.True(t, dekData.NeedsRotation) + + // if there no PendingDEK, and a NeedsRotation flag: NeedsRotation=true and + // GetKeys attempts to create a pending key and write it to disk. If successful, + // it retuns the new keys + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello")} + krw = ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + }) + dekManager, err = NewRaftDEKManager(krw) + + require.NoError(t, err) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + require.NoError(t, krw.Write(cert, key, nil)) + + require.True(t, dekManager.NeedsRotation()) + updatedKeys := dekManager.GetKeys() + require.Equal(t, keys.CurrentDEK, updatedKeys.CurrentDEK) + require.NotNil(t, updatedKeys.PendingDEK) + require.True(t, dekManager.NeedsRotation()) + + h, _ = krw.GetCurrentState() + dekData, ok = h.(RaftDEKData) + require.True(t, ok) + require.False(t, dekData.NeedsRotation) +} + +func TestRaftDEKManagerUpdateKeys(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-update-keys-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + keys := raft.EncryptionKeys{ + CurrentDEK: []byte("key1"), + PendingDEK: []byte("key2"), + } + krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + }) + require.NoError(t, krw.Write(cert, key, nil)) + + dekManager, err := NewRaftDEKManager(krw) + require.NoError(t, err) + + newKeys := raft.EncryptionKeys{ + CurrentDEK: []byte("new current"), + } + require.NoError(t, dekManager.UpdateKeys(newKeys)) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + + h, _ := krw.GetCurrentState() + dekData, ok := h.(RaftDEKData) + require.True(t, ok) + require.True(t, dekData.NeedsRotation) + + // UpdateKeys so there is no CurrentDEK: all the headers should be wiped out + require.NoError(t, dekManager.UpdateKeys(raft.EncryptionKeys{})) + require.Equal(t, raft.EncryptionKeys{}, dekManager.GetKeys()) + require.False(t, dekManager.NeedsRotation()) + + h, _ = krw.GetCurrentState() + require.Nil(t, h) + + keyBytes, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + keyBlock, _ := pem.Decode(keyBytes) + require.NotNil(t, keyBlock) + + // the only header remaining should be the kek version + require.Len(t, keyBlock.Headers, 1) + require.Contains(t, keyBlock.Headers, "kek-version") +} + +func TestRaftDEKManagerMaybeUpdateKEK(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-maybe-update-kek-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + keys := raft.EncryptionKeys{CurrentDEK: []byte("current dek")} + + // trying to update a KEK will error if the version is the same but the kek is different + krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{EncryptionKeys: keys}) + require.NoError(t, krw.Write(cert, key, nil)) + dekManager, err := NewRaftDEKManager(krw) + require.NoError(t, err) + + keyBytes, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + + _, _, err = dekManager.MaybeUpdateKEK(ca.KEKData{KEK: []byte("locked now")}) + require.Error(t, err) + require.False(t, dekManager.NeedsRotation()) + + keyBytes2, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2) + + // trying to update a KEK from unlocked to lock will set NeedsRotation to true, as well as encrypt the TLS key + updated, unlockedToLocked, err := dekManager.MaybeUpdateKEK(ca.KEKData{KEK: []byte("locked now"), Version: 1}) + require.NoError(t, err) + require.True(t, updated) + require.True(t, unlockedToLocked) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + h, _ := krw.GetCurrentState() + dekData, ok := h.(RaftDEKData) + require.True(t, ok) + require.Equal(t, keys, dekData.EncryptionKeys) + require.True(t, dekData.NeedsRotation) + require.NotNil(t, <-dekManager.RotationNotify()) // we are notified of a new pending key + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.NotEqual(t, keyBytes, keyBytes2) + keyBytes = keyBytes2 + + readKRW := ca.NewKeyReadWriter(paths.Node, []byte("locked now"), RaftDEKData{}) + _, _, err = readKRW.Read() + require.NoError(t, err) + + // trying to update a KEK of a lower version will not update anything, but will not error + updated, unlockedToLocked, err = dekManager.MaybeUpdateKEK(ca.KEKData{}) + require.NoError(t, err) + require.False(t, unlockedToLocked) + require.False(t, updated) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + h, _ = krw.GetCurrentState() + dekData, ok = h.(RaftDEKData) + require.True(t, ok) + require.Equal(t, keys, dekData.EncryptionKeys) + require.True(t, dekData.NeedsRotation) + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2, string(keyBytes), string(keyBytes2)) + + // updating a kek to a higher version, but with the same kek, will also neither update anything nor error + updated, unlockedToLocked, err = dekManager.MaybeUpdateKEK(ca.KEKData{KEK: []byte("locked now"), Version: 100}) + require.NoError(t, err) + require.False(t, unlockedToLocked) + require.False(t, updated) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + h, _ = krw.GetCurrentState() + dekData, ok = h.(RaftDEKData) + require.True(t, ok) + require.Equal(t, keys, dekData.EncryptionKeys) + require.True(t, dekData.NeedsRotation) + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2) + + // going from locked to unlock does not result in the NeedsRotation flag, but does result in + // the key being decrypted + krw = ca.NewKeyReadWriter(paths.Node, []byte("kek"), RaftDEKData{EncryptionKeys: keys}) + require.NoError(t, krw.Write(cert, key, nil)) + dekManager, err = NewRaftDEKManager(krw) + require.NoError(t, err) + + keyBytes, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + + updated, unlockedToLocked, err = dekManager.MaybeUpdateKEK(ca.KEKData{Version: 2}) + require.NoError(t, err) + require.False(t, unlockedToLocked) + require.True(t, updated) + require.Equal(t, keys, dekManager.GetKeys()) + require.False(t, dekManager.NeedsRotation()) + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.NotEqual(t, keyBytes, keyBytes2) + + readKRW = ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{}) + _, _, err = readKRW.Read() + require.NoError(t, err) +} + +// The TLS KEK and the KEK for the headers should be in sync, and so failing +// to decrypt the TLS key should be mean we won't be able to decrypt the headers. +// However, the TLS Key encryption uses AES-256-CBC (golang as of 1.7.x does not seem +// to support GCM, so no cipher modes with digests) so sometimes decrypting with +// the wrong passphrase will not result in an error. This means we will ultimately +// have to rely on the header encryption mechanism, which does include a digest, to +// determine if the KEK is valid. +func TestDecryptTLSKeyFalsePositive(t *testing.T) { + badKey := []byte(` +-----BEGIN EC PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,e7927e79e748233776c03c2eb7275f09 +kek-version: 392 +raft-dek: CAESMBrzZ0gNVPe3FRs42743q8RtkUBrK1ICQpHWX2vdQ8iqSKt1WoKdFDFD2r28LYAVLxoYQguwHbijMx9k+BALUNBAI3s199S5tvnr + +JfGenNvzm++AvsOh+UmcBY+JgI6lnfzaCB68agmlmEZYLYi5tqtAU7gif6VIJpCW ++Pj23Fzkw8sKKOOBeapSC5lp+Cjx9OsCci/R9xrdx+uxnnzKJNxOB/qzqcQfZDMh +id2LxdliFcPEk/Yj5gNGpT0UMFJ4G52enbOwOru46f0= +-----END EC PRIVATE KEY----- +`) + + // not actually a real swarm cert - generated a cert corresponding to the key that expires in 20 years + matchingCert := []byte(` +-----BEGIN CERTIFICATE----- +MIIB9jCCAZygAwIBAgIRAIdzF3Z9VT2OXbRvEw5cR68wCgYIKoZIzj0EAwIwYDEi +MCAGA1UEChMZbWRwMXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dh +cm0tbWFuYWdlcjEiMCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTAg +GA8wMDAxMDEwMTAwMDAwMFoXDTM2MTEwODA2MjMwMlowYDEiMCAGA1UEChMZbWRw +MXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dhcm0tbWFuYWdlcjEi +MCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTBZMBMGByqGSM49AgEG +CCqGSM49AwEHA0IABGOivD25E/zcupRFQdKOKbPHS9Mx7JlUhlWnl0iR0K5VhVIU +XjUHt98GuX6gDjs4yUzEKSGxYPsSYlnG9zQqbQSjNTAzMA4GA1UdDwEB/wQEAwIF +oDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMAoGCCqGSM49BAMC +A0gAMEUCIQDWtjg1ITGznQILipaEe70G/NgZAOtFfuPXTVkUl3el+wIgSVOVKB/Q +O0T3aXuZGYNyh//KqAoA3erCmh6HauMz84Y= +-----END CERTIFICATE----- + `) + + var wrongKEK []byte // empty passphrase doesn't decrypt without errors + falsePositiveKEK, err := base64.RawStdEncoding.DecodeString("bIQgLAAMoGCrHdjMLVhEVqnYTAM7ZNF2xWMiwtw7AiQ") + require.NoError(t, err) + realKEK, err := base64.RawStdEncoding.DecodeString("fDg9YejLnMjU+FpulWR62oJLzVpkD2j7VQuP5xiK9QA") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter-false-positive-decryption") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(tempdir) + require.NoError(t, ioutil.WriteFile(path.Node.Key, badKey, 0600)) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, matchingCert, 0644)) + + krw := ca.NewKeyReadWriter(path.Node, wrongKEK, RaftDEKData{}) + _, _, err = krw.Read() + require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err)) + + krw = ca.NewKeyReadWriter(path.Node, falsePositiveKEK, RaftDEKData{}) + _, _, err = krw.Read() + require.Error(t, err) + require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err)) + + krw = ca.NewKeyReadWriter(path.Node, realKEK, RaftDEKData{}) + _, _, err = krw.Read() + require.NoError(t, err) +} diff --git a/manager/encryption/encryption.go b/manager/encryption/encryption.go index 7ce834c1d7..313e0e2bbd 100644 --- a/manager/encryption/encryption.go +++ b/manager/encryption/encryption.go @@ -115,7 +115,7 @@ func GenerateSecretKey() []byte { // HumanReadableKey displays a secret key in a human readable way func HumanReadableKey(key []byte) string { // base64-encode the key - return humanReadablePrefix + base64.StdEncoding.EncodeToString(key) + return humanReadablePrefix + base64.RawStdEncoding.EncodeToString(key) } // ParseHumanReadableKey returns a key as bytes from recognized serializations of @@ -124,7 +124,7 @@ func ParseHumanReadableKey(key string) ([]byte, error) { if !strings.HasPrefix(key, humanReadablePrefix) { return nil, fmt.Errorf("invalid key string") } - keyBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(key, humanReadablePrefix)) + keyBytes, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(key, humanReadablePrefix)) if err != nil { return nil, fmt.Errorf("invalid key string") } diff --git a/manager/encryption/encryption_test.go b/manager/encryption/encryption_test.go index ef98a606b9..14f2d9a728 100644 --- a/manager/encryption/encryption_test.go +++ b/manager/encryption/encryption_test.go @@ -62,7 +62,7 @@ func TestHumanReadable(t *testing.T) { require.Error(t, err) // With the right prefix, we can't parse if the key isn't base64 encoded - _, err = ParseHumanReadableKey(humanReadablePrefix + "aaaaa/") + _, err = ParseHumanReadableKey(humanReadablePrefix + "aaa*aa/") require.Error(t, err) // Extra padding also fails diff --git a/manager/manager.go b/manager/manager.go index 0c10565423..975ca57fc5 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -29,9 +29,11 @@ import ( "github.com/docker/swarmkit/manager/orchestrator/taskreaper" "github.com/docker/swarmkit/manager/resourceapi" "github.com/docker/swarmkit/manager/scheduler" + "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/raft" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" + "github.com/docker/swarmkit/remotes" "github.com/docker/swarmkit/xnet" "github.com/pkg/errors" "golang.org/x/net/context" @@ -86,6 +88,17 @@ type Config struct { // HeartbeatTick defines the amount of ticks between each // heartbeat sent to other members for health-check purposes HeartbeatTick uint32 + + // AutoLockManagers determines whether or not managers require an unlock key + // when starting from a stopped state. This configuration parameter is only + // applicable when bootstrapping a new cluster for the first time. + AutoLockManagers bool + + // UnlockKey is the key to unlock a node - used for decrypting manager TLS keys + // as well as the raft data encryption key (DEK). It is applicable when + // bootstrapping a cluster for the first time (it's a cluster-wide setting), + // and also when loading up any raft data on disk (as a KEK for the raft DEK). + UnlockKey []byte } // Manager is the cluster manager for Swarm. @@ -108,6 +121,7 @@ type Manager struct { server *grpc.Server localserver *grpc.Server raftNode *raft.Node + dekRotator *RaftDEKManager cancelFunc context.CancelFunc @@ -217,6 +231,11 @@ func New(config *Config) (*Manager, error) { raftCfg.HeartbeatTick = int(config.HeartbeatTick) } + dekRotator, err := NewRaftDEKManager(config.SecurityConfig.KeyWriter()) + if err != nil { + return nil, err + } + newNodeOpts := raft.NodeOptions{ ID: config.SecurityConfig.ClientTLSCreds.NodeID(), Addr: advertiseAddr, @@ -225,6 +244,7 @@ func New(config *Config) (*Manager, error) { StateDir: raftStateDir, ForceNewCluster: config.ForceNewCluster, TLSCredentials: config.SecurityConfig.ClientTLSCreds, + KeyRotator: dekRotator, } raftNode := raft.NewNode(newNodeOpts) @@ -241,6 +261,7 @@ func New(config *Config) (*Manager, error) { localserver: grpc.NewServer(opts...), raftNode: raftNode, started: make(chan struct{}), + dekRotator: dekRotator, } return m, nil @@ -320,6 +341,7 @@ func (m *Manager) Run(parent context.Context) error { forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil } localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.raftNode, forwardAsOwnRequest) localProxyLogsAPI := api.NewRaftProxyLogsServer(m.logbroker, m.raftNode, forwardAsOwnRequest) + localCAAPI := api.NewRaftProxyCAServer(m.caserver, m.raftNode, forwardAsOwnRequest) // Everything registered on m.server should be an authenticated // wrapper, or a proxy wrapping an authenticated wrapper! @@ -337,6 +359,7 @@ func (m *Manager) Run(parent context.Context) error { api.RegisterControlServer(m.localserver, localProxyControlAPI) api.RegisterLogsServer(m.localserver, localProxyLogsAPI) api.RegisterHealthServer(m.localserver, localHealthServer) + api.RegisterCAServer(m.localserver, localCAAPI) healthServer.SetServingStatus("Raft", api.HealthCheckResponse_NOT_SERVING) localHealthServer.SetServingStatus("ControlAPI", api.HealthCheckResponse_NOT_SERVING) @@ -362,8 +385,12 @@ func (m *Manager) Run(parent context.Context) error { close(m.started) + watchDone := make(chan struct{}) + watchCtx, watchCtxCancel := context.WithCancel(parent) go func() { err := m.raftNode.Run(ctx) + watchCtxCancel() + <-watchDone if err != nil { log.G(ctx).Error(err) m.Stop(ctx) @@ -380,6 +407,10 @@ func (m *Manager) Run(parent context.Context) error { } raftConfig := c.Spec.Raft + if err := m.watchForKEKChanges(watchCtx, watchDone); err != nil { + return err + } + if int(raftConfig.ElectionTick) != m.raftNode.Config.ElectionTick { log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.ElectionTick, raftConfig.ElectionTick) } @@ -475,6 +506,78 @@ func (m *Manager) Stop(ctx context.Context) { // mutex is released and Run can return now } +func (m *Manager) updateKEK(ctx context.Context, cluster *api.Cluster) error { + securityConfig := m.config.SecurityConfig + nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID() + logger := log.G(ctx).WithFields(logrus.Fields{ + "node.id": nodeID, + "node.role": ca.ManagerRole, + }) + + // we are our own peer from which we get certs - try to connect over the local socket + r := remotes.NewRemotes(api.Peer{Addr: m.Addr(), NodeID: nodeID}) + + kekData := ca.KEKData{Version: cluster.Meta.Version.Index} + for _, encryptionKey := range cluster.UnlockKeys { + if encryptionKey.Subsystem == ca.ManagerRole { + kekData.KEK = encryptionKey.Key + break + } + } + updated, unlockedToLocked, err := m.dekRotator.MaybeUpdateKEK(kekData) + if err != nil { + logger.WithError(err).Errorf("failed to re-encrypt TLS key with a new KEK") + return err + } + if updated { + logger.Debug("successfully rotated KEK") + } + if unlockedToLocked { + // a best effort attempt to update the TLS certificate - if it fails, it'll be updated the next time it renews; + // don't wait because it might take a bit + go func() { + if err := ca.RenewTLSConfigNow(ctx, securityConfig, r); err != nil { + logger.WithError(err).Errorf("failed to download new TLS certificate after locking the cluster") + } + }() + } + return nil +} + +func (m *Manager) watchForKEKChanges(ctx context.Context, watchDone chan struct{}) error { + defer close(watchDone) + clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization() + clusterWatch, clusterWatchCancel, err := store.ViewAndWatch(m.raftNode.MemoryStore(), + func(tx store.ReadTx) error { + cluster := store.GetCluster(tx, clusterID) + if cluster == nil { + return fmt.Errorf("unable to get current cluster") + } + return m.updateKEK(ctx, cluster) + }, + state.EventUpdateCluster{ + Cluster: &api.Cluster{ID: clusterID}, + Checks: []state.ClusterCheckFunc{state.ClusterCheckID}, + }, + ) + if err != nil { + return err + } + go func() { + for { + select { + case event := <-clusterWatch: + clusterEvent := event.(state.EventUpdateCluster) + m.updateKEK(ctx, clusterEvent.Cluster) + case <-ctx.Done(): + clusterWatchCancel() + return + } + } + }() + return nil +} + // rotateRootCAKEK will attempt to rotate the key-encryption-key for root CA key-material in raft. // If there is no passphrase set in ENV, it returns. // If there is plain-text root key-material, and a passphrase set, it encrypts it. @@ -625,12 +728,26 @@ func (m *Manager) becomeLeader(ctx context.Context) { initialCAConfig := ca.DefaultCAConfig() initialCAConfig.ExternalCAs = m.config.ExternalCAs + var unlockKeys []*api.EncryptionKey + if m.config.AutoLockManagers { + unlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: m.config.UnlockKey, + }} + } + s.Update(func(tx store.Tx) error { // Add a default cluster object to the // store. Don't check the error because // we expect this to fail unless this // is a brand new cluster. - store.CreateCluster(tx, defaultClusterObject(clusterID, initialCAConfig, raftCfg, rootCA)) + store.CreateCluster(tx, defaultClusterObject( + clusterID, + initialCAConfig, + raftCfg, + api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers}, + unlockKeys, + rootCA)) // Add Node entry for ourself, if one // doesn't exist already. store.CreateNode(tx, managerNode(nodeID)) @@ -759,7 +876,14 @@ func (m *Manager) becomeFollower() { } // defaultClusterObject creates a default cluster. -func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCfg api.RaftConfig, rootCA *ca.RootCA) *api.Cluster { +func defaultClusterObject( + clusterID string, + initialCAConfig api.CAConfig, + raftCfg api.RaftConfig, + encryptionConfig api.EncryptionConfig, + initialUnlockKeys []*api.EncryptionKey, + rootCA *ca.RootCA) *api.Cluster { + return &api.Cluster{ ID: clusterID, Spec: api.ClusterSpec{ @@ -772,8 +896,9 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf Dispatcher: api.DispatcherConfig{ HeartbeatPeriod: ptypes.DurationProto(dispatcher.DefaultHeartBeatPeriod), }, - Raft: raftCfg, - CAConfig: initialCAConfig, + Raft: raftCfg, + CAConfig: initialCAConfig, + EncryptionConfig: encryptionConfig, }, RootCA: api.RootCA{ CAKey: rootCA.Key, @@ -784,6 +909,7 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf Manager: ca.GenerateJoinToken(rootCA), }, }, + UnlockKeys: initialUnlockKeys, } } diff --git a/manager/manager_test.go b/manager/manager_test.go index 990672a451..e34cf716d6 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -1,9 +1,14 @@ package manager import ( + "bytes" "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" "io/ioutil" "os" + "path/filepath" "testing" "time" @@ -16,8 +21,12 @@ import ( "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/ca/testutils" "github.com/docker/swarmkit/manager/dispatcher" + "github.com/docker/swarmkit/manager/encryption" + "github.com/docker/swarmkit/manager/state/raft/storage" + raftutils "github.com/docker/swarmkit/manager/state/raft/testutils" "github.com/docker/swarmkit/manager/state/store" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestManager(t *testing.T) { @@ -34,7 +43,9 @@ func TestManager(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(stateDir) - tc := testutils.NewTestCA(t) + tc := testutils.NewTestCA(t, func(p ca.CertPaths) *ca.KeyReadWriter { + return ca.NewKeyReadWriter(p, []byte("kek"), nil) + }) defer tc.Stop() agentSecurityConfig, err := tc.NewNodeConfig(ca.WorkerRole) @@ -45,10 +56,12 @@ func TestManager(t *testing.T) { assert.NoError(t, err) m, err := New(&Config{ - RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, - ControlAPI: temp.Name(), - StateDir: stateDir, - SecurityConfig: managerSecurityConfig, + RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, + ControlAPI: temp.Name(), + StateDir: stateDir, + SecurityConfig: managerSecurityConfig, + AutoLockManagers: true, + UnlockKey: []byte("kek"), }) assert.NoError(t, err) assert.NotNil(t, m) @@ -131,6 +144,21 @@ func TestManager(t *testing.T) { assert.NoError(t, controlConn.Close()) }() + // check that the kek is added to the config + var cluster api.Cluster + m.raftNode.MemoryStore().View(func(tx store.ReadTx) { + clusters, err := store.FindClusters(tx, store.All) + require.NoError(t, err) + require.Len(t, clusters, 1) + cluster = *clusters[0] + }) + require.NotNil(t, cluster) + require.Len(t, cluster.UnlockKeys, 1) + require.Equal(t, &api.EncryptionKey{ + Subsystem: ca.ManagerRole, + Key: []byte("kek"), + }, cluster.UnlockKeys[0]) + // Test removal of the agent node agentID := agentSecurityConfig.ClientTLSCreds.NodeID() assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error { @@ -164,3 +192,211 @@ func TestManager(t *testing.T) { // error. <-done } + +// Tests locking and unlocking the manager and key rotations +func TestManagerLockUnlock(t *testing.T) { + ctx := context.Background() + + temp, err := ioutil.TempFile("", "test-manager-lock") + require.NoError(t, err) + require.NoError(t, temp.Close()) + require.NoError(t, os.Remove(temp.Name())) + + defer os.RemoveAll(temp.Name()) + + stateDir, err := ioutil.TempDir("", "test-raft") + require.NoError(t, err) + defer os.RemoveAll(stateDir) + + tc := testutils.NewTestCA(t) + defer tc.Stop() + + managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole) + require.NoError(t, err) + + _, _, err = managerSecurityConfig.KeyReader().Read() + require.NoError(t, err) + + m, err := New(&Config{ + RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, + ControlAPI: temp.Name(), + StateDir: stateDir, + SecurityConfig: managerSecurityConfig, + // start off without any encryption + }) + require.NoError(t, err) + require.NotNil(t, m) + + done := make(chan error) + defer close(done) + go func() { + done <- m.Run(ctx) + }() + + opts := []grpc.DialOption{ + grpc.WithTimeout(10 * time.Second), + grpc.WithTransportCredentials(managerSecurityConfig.ClientTLSCreds), + } + + conn, err := grpc.Dial(m.Addr(), opts...) + require.NoError(t, err) + defer func() { + require.NoError(t, conn.Close()) + }() + + // check that there is no kek currently - we are using the API because this + // lets us wait until the manager is up and listening, as well + var cluster *api.Cluster + client := api.NewControlClient(conn) + + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + resp, err := client.ListClusters(ctx, &api.ListClustersRequest{}) + if err != nil { + return err + } + if len(resp.Clusters) == 0 { + return fmt.Errorf("no clusters yet") + } + cluster = resp.Clusters[0] + return nil + }, 1*time.Second)) + + require.Nil(t, cluster.UnlockKeys) + + // tls key is unencrypted, but there is a DEK + key, err := ioutil.ReadFile(tc.Paths.Node.Key) + require.NoError(t, err) + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) + require.Len(t, keyBlock.Headers, 2) + currentDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) + require.NoError(t, err) + require.NotEmpty(t, currentDEK) + + // update the lock key - this may fail due to update out of sequence errors, so try again + for { + getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) + require.NoError(t, err) + cluster = getResp.Cluster + + spec := cluster.Spec.Copy() + spec.EncryptionConfig.AutoLockManagers = true + updateResp, err := client.UpdateCluster(ctx, &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + ClusterVersion: &cluster.Meta.Version, + Spec: spec, + }) + if grpc.ErrorDesc(err) == "update out of sequence" { + continue + } + // if there is any other type of error, this should fail + if err == nil { + cluster = updateResp.Cluster + } + break + } + require.NoError(t, err) + + caConn := api.NewCAClient(conn) + unlockKeyResp, err := caConn.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{}) + require.NoError(t, err) + + // this should update the TLS key, rotate the DEK, and finish snapshotting + var updatedKey []byte + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + updatedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) + require.NoError(t, err) // this should never error due to atomic writes + + if bytes.Equal(key, updatedKey) { + return fmt.Errorf("TLS key should have been re-encrypted at least") + } + + keyBlock, _ = pem.Decode(updatedKey) + require.NotNil(t, keyBlock) // this should never error due to atomic writes + + if !x509.IsEncryptedPEMBlock(keyBlock) { + return fmt.Errorf("Key not encrypted") + } + + // we don't check that the TLS key has been rotated, because that may take + // a little bit, and is best effort only + + currentDEKString, ok := keyBlock.Headers[pemHeaderRaftDEK] + require.True(t, ok) // there should never NOT be a current header + nowCurrentDEK, err := decodePEMHeaderValue(currentDEKString, unlockKeyResp.UnlockKey) + require.NoError(t, err) // it should always be encrypted + if bytes.Equal(currentDEK, nowCurrentDEK) { + return fmt.Errorf("snapshot has not been finished yet") + } + + currentDEK = nowCurrentDEK + return nil + }, 1*time.Second)) + + _, ok := keyBlock.Headers[pemHeaderRaftPendingDEK] + require.False(t, ok) // once the snapshot is do + + _, ok = keyBlock.Headers[pemHeaderRaftDEKNeedsRotation] + require.False(t, ok) + + // verify that the snapshot is readable with the new DEK + encrypter, decrypter := encryption.Defaults(currentDEK) + // we can't use the raftLogger, because the WALs are still locked while the raft node is up. And once we remove + // the manager, they'll be deleted. + snapshot, err := storage.NewSnapFactory(encrypter, decrypter).New(filepath.Join(stateDir, "raft", "snap-v3-encrypted")).Load() + require.NoError(t, err) + require.NotNil(t, snapshot) + + // update the lock key to nil + for i := 0; i < 3; i++ { + getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) + require.NoError(t, err) + cluster = getResp.Cluster + + spec := cluster.Spec.Copy() + spec.EncryptionConfig.AutoLockManagers = false + _, err = client.UpdateCluster(ctx, &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + ClusterVersion: &cluster.Meta.Version, + Spec: spec, + }) + if grpc.ErrorDesc(err) == "update out of sequence" { + continue + } + require.NoError(t, err) + } + + // this should update the TLS key + var unlockedKey []byte + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + unlockedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) + if err != nil { + return err + } + + if bytes.Equal(unlockedKey, updatedKey) { + return fmt.Errorf("TLS key should have been rotated") + } + + return nil + }, 1*time.Second)) + + // the new key should not be encrypted, and the DEK should also be unencrypted + // but not rotated + keyBlock, _ = pem.Decode(unlockedKey) + require.NotNil(t, keyBlock) + require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) + + unencryptedDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) + require.NoError(t, err) + require.NotNil(t, unencryptedDEK) + require.Equal(t, currentDEK, unencryptedDEK) + + m.Stop(ctx) + + // After stopping we should MAY receive an error from ListenAndServe if + // all this happened before WaitForLeader completed, so don't check the + // error. + <-done +} diff --git a/manager/state/raft/raft.go b/manager/state/raft/raft.go index 42cf6d40e2..c3ff5a8432 100644 --- a/manager/state/raft/raft.go +++ b/manager/state/raft/raft.go @@ -20,14 +20,13 @@ import ( "github.com/coreos/etcd/pkg/idutil" "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" - "github.com/coreos/etcd/snap" - "github.com/coreos/etcd/wal" "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/raftselector" "github.com/docker/swarmkit/manager/state/raft/membership" + "github.com/docker/swarmkit/manager/state/raft/storage" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/watch" "github.com/gogo/protobuf/proto" @@ -75,6 +74,21 @@ const ( IsFollower ) +// EncryptionKeys are the current and, if necessary, pending DEKs with which to +// encrypt raft data +type EncryptionKeys struct { + CurrentDEK []byte + PendingDEK []byte +} + +// EncryptionKeyRotator is an interface to find out if any keys need rotating. +type EncryptionKeyRotator interface { + GetKeys() EncryptionKeys + UpdateKeys(EncryptionKeys) error + NeedsRotation() bool + RotationNotify() chan struct{} +} + // Node represents the Raft Node useful // configuration. type Node struct { @@ -87,8 +101,6 @@ type Node struct { opts NodeOptions reqIDGen *idutil.Generator wait *wait - wal *wal.WAL - snapshotter *snap.Snapshotter campaignWhenAble bool signalledLeadership uint32 isMember uint32 @@ -122,6 +134,9 @@ type Node struct { stopped chan struct{} lastSendToMember map[uint64]chan struct{} + raftLogger *storage.EncryptedRaftLogger + keyRotator EncryptionKeyRotator + rotationQueued bool } // NodeOptions provides node-level options. @@ -150,6 +165,8 @@ type NodeOptions struct { // nodes. Leave this as 0 to get the default value. SendTimeout time.Duration TLSCredentials credentials.TransportCredentials + + KeyRotator EncryptionKeyRotator } func init() { @@ -188,6 +205,7 @@ func NewNode(opts NodeOptions) *Node { stopped: make(chan struct{}), leadershipBroadcast: watch.NewQueue(), lastSendToMember: make(map[uint64]chan struct{}), + keyRotator: opts.KeyRotator, } n.memoryStore = store.NewMemoryStore(n) @@ -238,7 +256,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) { }() loadAndStartErr := n.loadAndStart(ctx, n.opts.ForceNewCluster) - if loadAndStartErr != nil && loadAndStartErr != errNoWAL { + if loadAndStartErr != nil && loadAndStartErr != storage.ErrNoWAL { return loadAndStartErr } @@ -252,7 +270,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) { n.appliedIndex = snapshot.Metadata.Index n.snapshotIndex = snapshot.Metadata.Index - if loadAndStartErr == errNoWAL { + if loadAndStartErr == storage.ErrNoWAL { if n.opts.JoinAddr != "" { c, err := n.ConnectToMember(n.opts.JoinAddr, 10*time.Second) if err != nil { @@ -274,22 +292,20 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) { n.Config.ID = resp.RaftID - if _, err := n.createWAL(n.opts.ID); err != nil { + if _, err := n.newRaftLogs(n.opts.ID); err != nil { return err } n.raftNode = raft.StartNode(n.Config, []raft.Peer{}) if err := n.registerNodes(resp.Members); err != nil { - if walErr := n.wal.Close(); err != nil { - log.G(ctx).WithError(walErr).Error("raft: error closing WAL") - } + n.raftLogger.Close(ctx) return err } } else { // First member in the cluster, self-assign ID n.Config.ID = uint64(rand.Int63()) + 1 - peer, err := n.createWAL(n.opts.ID) + peer, err := n.newRaftLogs(n.opts.ID) if err != nil { return err } @@ -367,9 +383,13 @@ func (n *Node) Run(ctx context.Context) error { if nodeRemoved { // Move WAL and snapshot out of the way, since // they are no longer usable. - if err := n.moveWALAndSnap(); err != nil { + if err := n.raftLogger.Clear(ctx); err != nil { log.G(ctx).WithError(err).Error("failed to move wal after node removal") } + // clear out the DEKs + if err := n.keyRotator.UpdateKeys(EncryptionKeys{}); err != nil { + log.G(ctx).WithError(err).Error("could not remove DEKs") + } } n.done() }() @@ -382,16 +402,10 @@ func (n *Node) Run(ctx context.Context) error { n.raftNode.Tick() n.cluster.Tick() case rd := <-n.raftNode.Ready(): - raftConfig := DefaultRaftConfig() - n.memoryStore.View(func(readTx store.ReadTx) { - clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) - if err == nil && len(clusters) == 1 { - raftConfig = clusters[0].Spec.Raft - } - }) + raftConfig := n.getCurrentRaftConfig() // Save entries to storage - if err := n.saveToStorage(&raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil { + if err := n.saveToStorage(ctx, &raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil { log.G(ctx).WithError(err).Error("failed to save entries to storage") } @@ -459,8 +473,8 @@ func (n *Node) Run(ctx context.Context) error { // Trigger a snapshot every once in awhile if n.snapshotInProgress == nil && - raftConfig.SnapshotInterval > 0 && - n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval { + (n.keyRotator.NeedsRotation() || raftConfig.SnapshotInterval > 0 && + n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval) { n.doSnapshot(ctx, raftConfig) } @@ -496,6 +510,24 @@ func (n *Node) Run(ctx context.Context) error { n.snapshotIndex = snapshotIndex } n.snapshotInProgress = nil + if n.rotationQueued { + // there was a key rotation that took place before while the snapshot + // was in progress - we have to take another snapshot and encrypt with the new key + n.doSnapshot(ctx, n.getCurrentRaftConfig()) + } + case <-n.keyRotator.RotationNotify(): + // There are 2 separate checks: rotationQueued, and keyRotator.NeedsRotation(). + // We set rotationQueued so that when we are notified of a rotation, we try to + // do a snapshot as soon as possible. However, if there is an error while doing + // the snapshot, we don't want to hammer the node attempting to do snapshots over + // and over. So if doing a snapshot fails, wait until the next entry comes in to + // try again. + switch { + case n.snapshotInProgress != nil: + n.rotationQueued = true + case n.keyRotator.NeedsRotation(): + n.doSnapshot(ctx, n.getCurrentRaftConfig()) + } case <-n.removeRaftCh: nodeRemoved = true // If the node was removed from other members, @@ -508,6 +540,17 @@ func (n *Node) Run(ctx context.Context) error { } } +func (n *Node) getCurrentRaftConfig() api.RaftConfig { + raftConfig := DefaultRaftConfig() + n.memoryStore.View(func(readTx store.ReadTx) { + clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) + if err == nil && len(clusters) == 1 { + raftConfig = clusters[0].Spec.Raft + } + }) + return raftConfig +} + // Done returns channel which is closed when raft node is fully stopped. func (n *Node) Done() <-chan struct{} { return n.doneCh @@ -524,9 +567,7 @@ func (n *Node) stop(ctx context.Context) { n.raftNode.Stop() n.ticker.Stop() - if err := n.wal.Close(); err != nil { - log.G(ctx).WithError(err).Error("raft: failed to close WAL") - } + n.raftLogger.Close(ctx) atomic.StoreUint32(&n.isMember, 0) // TODO(stevvooe): Handle ctx.Done() } @@ -1123,17 +1164,27 @@ func (n *Node) canSubmitProposal() bool { } // Saves a log entry to our Store -func (n *Node) saveToStorage(raftConfig *api.RaftConfig, hardState raftpb.HardState, entries []raftpb.Entry, snapshot raftpb.Snapshot) (err error) { +func (n *Node) saveToStorage( + ctx context.Context, + raftConfig *api.RaftConfig, + hardState raftpb.HardState, + entries []raftpb.Entry, + snapshot raftpb.Snapshot, +) (err error) { + if !raft.IsEmptySnap(snapshot) { - if err := n.saveSnapshot(snapshot, raftConfig.KeepOldSnapshots); err != nil { + if err := n.raftLogger.SaveSnapshot(snapshot); err != nil { return ErrApplySnapshot } + if err := n.raftLogger.GC(snapshot.Metadata.Index, snapshot.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil { + log.G(ctx).WithError(err).Error("unable to clean old snapshots and WALs") + } if err = n.raftStore.ApplySnapshot(snapshot); err != nil { return ErrApplySnapshot } } - if err := n.wal.Save(hardState, entries); err != nil { + if err := n.raftLogger.SaveEntries(hardState, entries); err != nil { // TODO(aaronl): These error types should really wrap more // detailed errors. return ErrApplySnapshot diff --git a/manager/state/raft/raft_test.go b/manager/state/raft/raft_test.go index 7ff390a6c0..c91336b2ba 100644 --- a/manager/state/raft/raft_test.go +++ b/manager/state/raft/raft_test.go @@ -6,6 +6,7 @@ import ( "log" "math/rand" "os" + "path/filepath" "reflect" "strconv" "testing" @@ -317,6 +318,13 @@ func TestRaftLeaderLeave(t *testing.T) { // Wait for election tick raftutils.WaitForCluster(t, clockSource, newCluster) + // Node1's state should be cleared + _, err = os.Stat(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted")) + require.True(t, os.IsNotExist(err)) + _, err = os.Stat(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted")) + require.True(t, os.IsNotExist(err)) + require.Equal(t, raft.EncryptionKeys{}, nodes[1].KeyRotator.GetKeys()) + // Leader should not be 1 assert.NotEqual(t, nodes[2].Leader(), nodes[1].Config.ID) assert.Equal(t, nodes[2].Leader(), nodes[3].Leader()) diff --git a/manager/state/raft/storage.go b/manager/state/raft/storage.go index c47da6ce68..8a7dd422c1 100644 --- a/manager/state/raft/storage.go +++ b/manager/state/raft/storage.go @@ -2,282 +2,72 @@ package raft import ( "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/coreos/etcd/pkg/fileutil" + "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" - "github.com/coreos/etcd/snap" - "github.com/coreos/etcd/wal" - "github.com/coreos/etcd/wal/walpb" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/manager/state/raft/membership" + "github.com/docker/swarmkit/manager/state/raft/storage" "github.com/docker/swarmkit/manager/state/store" "github.com/pkg/errors" "golang.org/x/net/context" ) -var errNoWAL = errors.New("no WAL present") - -func (n *Node) legacyWALDir() string { - return filepath.Join(n.opts.StateDir, "wal") -} - -func (n *Node) walDir() string { - return filepath.Join(n.opts.StateDir, "wal-v3") -} - -func (n *Node) legacySnapDir() string { - return filepath.Join(n.opts.StateDir, "snap") -} - -func (n *Node) snapDir() string { - return filepath.Join(n.opts.StateDir, "snap-v3") -} - -func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error { - walDir := n.walDir() - snapDir := n.snapDir() - - if !fileutil.Exist(snapDir) { - // If snapshots created by the etcd-v2 code exist, hard link - // them at the new path. This prevents etc-v2 creating - // snapshots that are visible to us, but out of sync with our - // WALs, after a downgrade. - legacySnapDir := n.legacySnapDir() - if fileutil.Exist(legacySnapDir) { - if err := migrateSnapshots(legacySnapDir, snapDir); err != nil { - return err - } - } else if err := os.MkdirAll(snapDir, 0700); err != nil { - return errors.Wrap(err, "failed to create snapshot directory") - } - } - - // Create a snapshotter - n.snapshotter = snap.New(snapDir) - - if !wal.Exist(walDir) { - // If wals created by the etcd-v2 wal code exist, copy them to - // the new path to avoid adding backwards-incompatible entries - // to those files. - legacyWALDir := n.legacyWALDir() - if !wal.Exist(legacyWALDir) { - return errNoWAL - } - - if err := migrateWALs(legacyWALDir, walDir); err != nil { - return err - } - } - - // Load snapshot data - snapshot, err := n.snapshotter.Load() - if err != nil && err != snap.ErrNoSnapshot { - return err - } - - if snapshot != nil { - // Load the snapshot data into the store - if err := n.restoreFromSnapshot(snapshot.Data, forceNewCluster); err != nil { - return err - } - } - - // Read logs to fully catch up store - if err := n.readWAL(ctx, snapshot, forceNewCluster); err != nil { - return err - } - - return nil -} +func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) { + keys := n.keyRotator.GetKeys() -func migrateWALs(legacyWALDir, walDir string) error { - // keep temporary wal directory so WAL initialization appears atomic - tmpdirpath := filepath.Clean(walDir) + ".tmp" - if fileutil.Exist(tmpdirpath) { - if err := os.RemoveAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not remove temporary WAL directory") - } - } - if err := fileutil.CreateDirAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not create temporary WAL directory") + n.raftLogger = &storage.EncryptedRaftLogger{ + StateDir: n.opts.StateDir, + EncryptionKey: keys.CurrentDEK, } - - walNames, err := fileutil.ReadDir(legacyWALDir) - if err != nil { - return errors.Wrapf(err, "could not list WAL directory %s", legacyWALDir) + if keys.PendingDEK != nil { + n.raftLogger.EncryptionKey = keys.PendingDEK } - for _, fname := range walNames { - _, err := copyFile(filepath.Join(legacyWALDir, fname), filepath.Join(tmpdirpath, fname), 0600) - if err != nil { - return errors.Wrap(err, "error copying WAL file") - } - } + snap, walData, err := n.raftLogger.BootstrapFromDisk(ctx) - if err := os.Rename(tmpdirpath, walDir); err != nil { - return err - } - - return nil -} - -func migrateSnapshots(legacySnapDir, snapDir string) error { - // use temporary snaphot directory so initialization appears atomic - tmpdirpath := filepath.Clean(snapDir) + ".tmp" - if fileutil.Exist(tmpdirpath) { - if err := os.RemoveAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not remove temporary snapshot directory") - } - } - if err := fileutil.CreateDirAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not create temporary snapshot directory") - } - - snapshotNames, err := fileutil.ReadDir(legacySnapDir) - if err != nil { - return errors.Wrapf(err, "could not list snapshot directory %s", legacySnapDir) - } - - for _, fname := range snapshotNames { - err := os.Link(filepath.Join(legacySnapDir, fname), filepath.Join(tmpdirpath, fname)) - if err != nil { - return errors.Wrap(err, "error linking snapshot file") + if keys.PendingDEK != nil { + switch errors.Cause(err).(type) { + case nil: + if err = n.keyRotator.UpdateKeys(EncryptionKeys{CurrentDEK: keys.PendingDEK}); err != nil { + err = errors.Wrap(err, "previous key rotation was successful, but unable mark rotation as complete") + } + case encryption.ErrCannotDecrypt: + snap, walData, err = n.raftLogger.BootstrapFromDisk(ctx, keys.CurrentDEK) } } - if err := os.Rename(tmpdirpath, snapDir); err != nil { - return err - } - - return nil -} - -// copyFile copies from src to dst until either EOF is reached -// on src or an error occurs. It verifies src exists and removes -// the dst if it exists. -func copyFile(src, dst string, perm os.FileMode) (int64, error) { - cleanSrc := filepath.Clean(src) - cleanDst := filepath.Clean(dst) - if cleanSrc == cleanDst { - return 0, nil - } - sf, err := os.Open(cleanSrc) - if err != nil { - return 0, err - } - defer sf.Close() - if err := os.Remove(cleanDst); err != nil && !os.IsNotExist(err) { - return 0, err - } - df, err := os.OpenFile(cleanDst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm) if err != nil { - return 0, err + return nil, storage.WALData{}, err } - defer df.Close() - return io.Copy(df, sf) + return snap, walData, nil } -func (n *Node) createWAL(nodeID string) (raft.Peer, error) { - raftNode := &api.RaftMember{ - RaftID: n.Config.ID, - NodeID: nodeID, - Addr: n.opts.Addr, - } - metadata, err := raftNode.Marshal() - if err != nil { - return raft.Peer{}, errors.Wrap(err, "error marshalling raft node") - } - n.wal, err = wal.Create(n.walDir(), metadata) - if err != nil { - return raft.Peer{}, errors.Wrap(err, "failed to create WAL") - } - - n.cluster.AddMember(&membership.Member{RaftMember: raftNode}) - return raft.Peer{ID: n.Config.ID, Context: metadata}, nil -} - -// moveWALAndSnap moves away the WAL and snapshot because we were removed -// from the cluster and will need to recreate them if we are readded. -func (n *Node) moveWALAndSnap() error { - newWALDir, err := ioutil.TempDir(n.opts.StateDir, "wal.") - if err != nil { - return err - } - err = os.Rename(n.walDir(), newWALDir) - if err != nil { - return err - } - - newSnapDir, err := ioutil.TempDir(n.opts.StateDir, "snap.") - if err != nil { - return err - } - err = os.Rename(n.snapDir(), newSnapDir) +// bootstraps a node's raft store from the raft logs and snapshots on disk +func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error { + snapshot, waldata, err := n.readFromDisk(ctx) if err != nil { return err } - return nil -} - -func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewCluster bool) (err error) { - var ( - walsnap walpb.Snapshot - metadata []byte - st raftpb.HardState - ents []raftpb.Entry - ) - if snapshot != nil { - walsnap.Index = snapshot.Metadata.Index - walsnap.Term = snapshot.Metadata.Term - } - - repaired := false - for { - if n.wal, err = wal.Open(n.walDir(), walsnap); err != nil { - return errors.Wrap(err, "failed to open WAL") - } - if metadata, st, ents, err = n.wal.ReadAll(); err != nil { - if err := n.wal.Close(); err != nil { - return err - } - // we can only repair ErrUnexpectedEOF and we never repair twice. - if repaired || err != io.ErrUnexpectedEOF { - return errors.Wrap(err, "irreparable WAL error") - } - if !wal.Repair(n.walDir()) { - return errors.Wrap(err, "WAL error cannot be repaired") - } - log.G(ctx).WithError(err).Info("repaired WAL error") - repaired = true - continue + // Load the snapshot data into the store + if err := n.restoreFromSnapshot(snapshot.Data, forceNewCluster); err != nil { + return err } - break } - defer func() { - if err != nil { - if walErr := n.wal.Close(); walErr != nil { - log.G(ctx).WithError(walErr).Error("error closing raft WAL") - } - } - }() - + // Read logs to fully catch up store var raftNode api.RaftMember - if err := raftNode.Unmarshal(metadata); err != nil { + if err := raftNode.Unmarshal(waldata.Metadata); err != nil { return errors.Wrap(err, "failed to unmarshal WAL metadata") } n.Config.ID = raftNode.RaftID + ents, st := waldata.Entries, waldata.HardState + // All members that are no longer part of the cluster must be added to // the removed list right away, so that we don't try to connect to them // before processing the configuration change entries, which could make @@ -326,7 +116,7 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC ents = append(ents, toAppEnts...) // force commit newly appended entries - err := n.wal.Save(st, toAppEnts) + err := n.raftLogger.SaveEntries(st, toAppEnts) if err != nil { log.G(ctx).WithError(err).Fatalf("failed to save WAL while forcing new cluster") } @@ -343,146 +133,24 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC if err := n.raftStore.SetHardState(st); err != nil { return err } - if err := n.raftStore.Append(ents); err != nil { - return err - } - - return nil + return n.raftStore.Append(ents) } -func (n *Node) saveSnapshot(snapshot raftpb.Snapshot, keepOldSnapshots uint64) error { - err := n.wal.SaveSnapshot(walpb.Snapshot{ - Index: snapshot.Metadata.Index, - Term: snapshot.Metadata.Term, - }) - if err != nil { - return err - } - err = n.snapshotter.SaveSnap(snapshot) - if err != nil { - return err - } - err = n.wal.ReleaseLockTo(snapshot.Metadata.Index) - if err != nil { - return err - } - - // Delete any older snapshots - curSnapshot := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, ".snap") - - dirents, err := ioutil.ReadDir(n.snapDir()) - if err != nil { - return err - } - - var snapshots []string - for _, dirent := range dirents { - if strings.HasSuffix(dirent.Name(), ".snap") { - snapshots = append(snapshots, dirent.Name()) - } - } - - // Sort snapshot filenames in reverse lexical order - sort.Sort(sort.Reverse(sort.StringSlice(snapshots))) - - // Ignore any snapshots that are older than the current snapshot. - // Delete the others. Rather than doing lexical comparisons, we look - // at what exists before/after the current snapshot in the slice. - // This means that if the current snapshot doesn't appear in the - // directory for some strange reason, we won't delete anything, which - // is the safe behavior. - curSnapshotIdx := -1 - var ( - removeErr error - oldestSnapshot string - ) - - for i, snapFile := range snapshots { - if curSnapshotIdx >= 0 && i > curSnapshotIdx { - if uint64(i-curSnapshotIdx) > keepOldSnapshots { - err := os.Remove(filepath.Join(n.snapDir(), snapFile)) - if err != nil && removeErr == nil { - removeErr = err - } - continue - } - } else if snapFile == curSnapshot { - curSnapshotIdx = i - } - oldestSnapshot = snapFile - } - - if removeErr != nil { - return removeErr - } - - // Remove any WAL files that only contain data from before the oldest - // remaining snapshot. - - if oldestSnapshot == "" { - return nil - } - - // Parse index out of oldest snapshot's filename - var snapTerm, snapIndex uint64 - _, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex) - if err != nil { - return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot) +func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) { + raftNode := &api.RaftMember{ + RaftID: n.Config.ID, + NodeID: nodeID, + Addr: n.opts.Addr, } - - // List the WALs - dirents, err = ioutil.ReadDir(n.walDir()) + metadata, err := raftNode.Marshal() if err != nil { - return err - } - - var wals []string - for _, dirent := range dirents { - if strings.HasSuffix(dirent.Name(), ".wal") { - wals = append(wals, dirent.Name()) - } - } - - // Sort WAL filenames in lexical order - sort.Sort(sort.StringSlice(wals)) - - found := false - deleteUntil := -1 - - for i, walName := range wals { - var walSeq, walIndex uint64 - _, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex) - if err != nil { - return errors.Wrapf(err, "could not parse WAL name %s", walName) - } - - if walIndex >= snapIndex { - deleteUntil = i - 1 - found = true - break - } - } - - // If all WAL files started with indices below the oldest snapshot's - // index, we can delete all but the newest WAL file. - if !found && len(wals) != 0 { - deleteUntil = len(wals) - 1 + return raft.Peer{}, errors.Wrap(err, "error marshalling raft node") } - - for i := 0; i < deleteUntil; i++ { - walPath := filepath.Join(n.walDir(), wals[i]) - l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode) - if err != nil { - return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i]) - } - err = os.Remove(walPath) - l.Close() - if err != nil { - return errors.Wrapf(err, "error removing old WAL file %s", wals[i]) - } + if err := n.raftLogger.BootstrapNew(metadata); err != nil { + return raft.Peer{}, err } - - return nil + n.cluster.AddMember(&membership.Member{RaftMember: raftNode}) + return raft.Peer{ID: n.Config.ID, Context: metadata}, nil } func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { @@ -497,6 +165,17 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { } snapshot.Membership.Removed = n.cluster.Removed() + // maybe start rotation + n.rotationQueued = false + var newEncryptionKeys *EncryptionKeys + if n.keyRotator.NeedsRotation() { + keys := n.keyRotator.GetKeys() + if keys.PendingDEK != nil { + n.raftLogger.RotateEncryptionKey(keys.PendingDEK) + newEncryptionKeys = &EncryptionKeys{CurrentDEK: keys.PendingDEK} + } + } + viewStarted := make(chan struct{}) n.asyncTasks.Add(1) n.snapshotInProgress = make(chan uint64, 1) // buffered in case Shutdown is called during the snapshot @@ -505,7 +184,6 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { n.asyncTasks.Done() n.snapshotInProgress <- snapshotIndex }() - var err error n.memoryStore.View(func(tx store.ReadTx) { close(viewStarted) @@ -526,11 +204,18 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { } snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d) if err == nil { - if err := n.saveSnapshot(snap, raftConfig.KeepOldSnapshots); err != nil { + if err := n.raftLogger.SaveSnapshot(snap); err != nil { log.G(ctx).WithError(err).Error("failed to save snapshot") return } snapshotIndex = appliedIndex + if newEncryptionKeys != nil { + // this means we tried to rotate - so finish the rotation + if err := n.keyRotator.UpdateKeys(*newEncryptionKeys); err != nil { + log.G(ctx).WithError(err).Error( + "failed to update encryption keys after a rotation - will wait for the next snapshot") + } + } if appliedIndex > raftConfig.LogEntriesForSlowFollowers { err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers) @@ -538,6 +223,10 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { log.G(ctx).WithError(err).Error("failed to compact snapshot") } } + + if err := n.raftLogger.GC(snap.Metadata.Index, snap.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil { + log.G(ctx).WithError(err).Error("failed to clean up old snapshots and WALs") + } } else if err != raft.ErrSnapOutOfDate { log.G(ctx).WithError(err).Error("failed to create snapshot") } diff --git a/manager/state/raft/storage/snapwrap.go b/manager/state/raft/storage/snapwrap.go index 08f475fb5f..05acbab4f0 100644 --- a/manager/state/raft/storage/snapwrap.go +++ b/manager/state/raft/storage/snapwrap.go @@ -1,9 +1,17 @@ package storage import ( + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/raft/raftpb" "github.com/coreos/etcd/snap" "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" ) // This package wraps the github.com/coreos/etcd/snap package, and encrypts @@ -57,6 +65,7 @@ func (s *wrappedSnap) Load() (*raftpb.Snapshot, error) { if err != nil { return nil, err } + return snapshot, nil } @@ -93,3 +102,57 @@ func (o originalSnap) New(dirpath string) Snapshotter { // OriginalSnap is the original `snap` package as an implemntation of the SnapFactory interface var OriginalSnap SnapFactory = originalSnap{} + +// MigrateSnapshot reads the latest existing snapshot from one directory, encoded one way, and writes +// it to a new directory, encoded a different way +func MigrateSnapshot(oldDir, newDir string, oldFactory, newFactory SnapFactory) error { + // use temporary snapshot directory so initialization appears atomic + oldSnapshotter := oldFactory.New(oldDir) + snapshot, err := oldSnapshotter.Load() + switch err { + case snap.ErrNoSnapshot: // if there's no snapshot, the migration succeeded + return nil + case nil: + break + default: + return err + } + + tmpdirpath := filepath.Clean(newDir) + ".tmp" + if fileutil.Exist(tmpdirpath) { + if err := os.RemoveAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not remove temporary snapshot directory") + } + } + if err := fileutil.CreateDirAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not create temporary snapshot directory") + } + tmpSnapshotter := newFactory.New(tmpdirpath) + + // write the new snapshot to the temporary location + if err = tmpSnapshotter.SaveSnap(*snapshot); err != nil { + return err + } + + return os.Rename(tmpdirpath, newDir) +} + +// ListSnapshots lists all the snapshot files in a particular directory and returns +// the snapshot files in reverse lexical order (newest first) +func ListSnapshots(dirpath string) ([]string, error) { + dirents, err := ioutil.ReadDir(dirpath) + if err != nil { + return nil, err + } + + var snapshots []string + for _, dirent := range dirents { + if strings.HasSuffix(dirent.Name(), ".snap") { + snapshots = append(snapshots, dirent.Name()) + } + } + + // Sort snapshot filenames in reverse lexical order + sort.Sort(sort.Reverse(sort.StringSlice(snapshots))) + return snapshots, nil +} diff --git a/manager/state/raft/storage/snapwrap_test.go b/manager/state/raft/storage/snapwrap_test.go index ebee634080..01e10ed4d4 100644 --- a/manager/state/raft/storage/snapwrap_test.go +++ b/manager/state/raft/storage/snapwrap_test.go @@ -1,6 +1,7 @@ package storage import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -174,3 +175,59 @@ func TestSaveAndLoad(t *testing.T) { require.NoError(t, err) require.Equal(t, fakeSnapshotData, *readSnap) } + +func TestMigrateSnapshot(t *testing.T) { + crypter := &meowCrypter{} + c := NewSnapFactory(crypter, crypter) + var ( + err error + dirs = make([]string, 3) + ) + + tempDir, err := ioutil.TempDir("", "test-migrate") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + for i := range dirs { + dirs[i] = filepath.Join(tempDir, fmt.Sprintf("snapDir%d", i)) + } + require.NoError(t, os.Mkdir(dirs[0], 0755)) + require.NoError(t, OriginalSnap.New(dirs[0]).SaveSnap(fakeSnapshotData)) + + // original to new + oldDir := dirs[0] + newDir := dirs[1] + + err = MigrateSnapshot(oldDir, newDir, OriginalSnap, c) + require.NoError(t, err) + + readSnap, err := c.New(newDir).Load() + require.NoError(t, err) + require.Equal(t, fakeSnapshotData, *readSnap) + + // new to original + oldDir = dirs[1] + newDir = dirs[2] + + err = MigrateSnapshot(oldDir, newDir, c, OriginalSnap) + require.NoError(t, err) + + readSnap, err = OriginalSnap.New(newDir).Load() + require.NoError(t, err) + require.Equal(t, fakeSnapshotData, *readSnap) + + // We can migrate from empty directory without error + for _, dir := range dirs { + require.NoError(t, os.RemoveAll(dir)) + } + require.NoError(t, os.Mkdir(dirs[0], 0755)) + oldDir = dirs[0] + newDir = dirs[1] + + err = MigrateSnapshot(oldDir, newDir, OriginalSnap, c) + require.NoError(t, err) + + subdirs, err := ioutil.ReadDir(tempDir) + require.NoError(t, err) + require.Len(t, subdirs, 1) +} diff --git a/manager/state/raft/storage/storage.go b/manager/state/raft/storage/storage.go new file mode 100644 index 0000000000..d830767ded --- /dev/null +++ b/manager/state/raft/storage/storage.go @@ -0,0 +1,391 @@ +package storage + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "golang.org/x/net/context" + + "github.com/coreos/etcd/pkg/fileutil" + "github.com/coreos/etcd/raft/raftpb" + "github.com/coreos/etcd/snap" + "github.com/coreos/etcd/wal" + "github.com/coreos/etcd/wal/walpb" + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" +) + +// ErrNoWAL is returned if there are no WALs on disk +var ErrNoWAL = errors.New("no WAL present") + +type walSnapDirs struct { + wal string + snap string +} + +// the wal/snap directories in decreasing order of preference/version +var versionedWALSnapDirs = []walSnapDirs{ + {wal: "wal-v3-encrypted", snap: "snap-v3-encrypted"}, + {wal: "wal-v3", snap: "snap-v3"}, + {wal: "wal", snap: "snap"}, +} + +// MultiDecrypter attempts to decrypt with a list of decrypters +type MultiDecrypter []encryption.Decrypter + +// Decrypt tries to decrypt using all the decrypters +func (m MultiDecrypter) Decrypt(r api.MaybeEncryptedRecord) (result []byte, err error) { + for _, d := range m { + result, err = d.Decrypt(r) + if err == nil { + return + } + } + return +} + +// EncryptedRaftLogger saves raft data to disk +type EncryptedRaftLogger struct { + StateDir string + EncryptionKey []byte + + // mutex is locked for writing only when we need to replace the wal object and snapshotter + // object, not when we're writing snapshots or wals (in which case it's locked for reading) + encoderMu sync.RWMutex + wal WAL + snapshotter Snapshotter +} + +// BootstrapFromDisk creates a new snapshotter and wal, and also reads the latest snapshot and WALs from disk +func (e *EncryptedRaftLogger) BootstrapFromDisk(ctx context.Context, oldEncryptionKeys ...[]byte) (*raftpb.Snapshot, WALData, error) { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + walDir := e.walDir() + snapDir := e.snapDir() + + encrypter, decrypter := encryption.Defaults(e.EncryptionKey) + if oldEncryptionKeys != nil { + decrypters := []encryption.Decrypter{decrypter} + for _, key := range oldEncryptionKeys { + _, d := encryption.Defaults(key) + decrypters = append(decrypters, d) + } + decrypter = MultiDecrypter(decrypters) + } + + snapFactory := NewSnapFactory(encrypter, decrypter) + + if !fileutil.Exist(snapDir) { + // If snapshots created by the etcd-v2 code exist, or by swarmkit development version, + // read the latest snapshot and write it encoded to the new path. The new path + // prevents etc-v2 creating snapshots that are visible to us, but not encoded and + // out of sync with our WALs, after a downgrade. + for _, dirs := range versionedWALSnapDirs[1:] { + legacySnapDir := filepath.Join(e.StateDir, dirs.snap) + if fileutil.Exist(legacySnapDir) { + if err := MigrateSnapshot(legacySnapDir, snapDir, OriginalSnap, snapFactory); err != nil { + return nil, WALData{}, err + } + break + } + } + } + // ensure the new directory exists + if err := os.MkdirAll(snapDir, 0700); err != nil { + return nil, WALData{}, errors.Wrap(err, "failed to create snapshot directory") + } + + var ( + snapshotter Snapshotter + walObj WAL + err error + ) + + // Create a snapshotter and load snapshot data + snapshotter = snapFactory.New(snapDir) + snapshot, err := snapshotter.Load() + if err != nil && err != snap.ErrNoSnapshot { + return nil, WALData{}, err + } + + walFactory := NewWALFactory(encrypter, decrypter) + var walsnap walpb.Snapshot + if snapshot != nil { + walsnap.Index = snapshot.Metadata.Index + walsnap.Term = snapshot.Metadata.Term + } + + if !wal.Exist(walDir) { + var walExists bool + // If wals created by the etcd-v2 wal code exist, read the latest ones based + // on this snapshot and encode them to wals in the new path to avoid adding + // backwards-incompatible entries to those files. + for _, dirs := range versionedWALSnapDirs[1:] { + legacyWALDir := filepath.Join(e.StateDir, dirs.wal) + if !wal.Exist(legacyWALDir) { + continue + } + if err = MigrateWALs(ctx, legacyWALDir, walDir, OriginalWAL, walFactory, walsnap); err != nil { + return nil, WALData{}, err + } + walExists = true + break + } + if !walExists { + return nil, WALData{}, ErrNoWAL + } + } + + walObj, waldata, err := ReadRepairWAL(ctx, walDir, walsnap, walFactory) + if err != nil { + return nil, WALData{}, err + } + + e.snapshotter = snapshotter + e.wal = walObj + + return snapshot, waldata, nil +} + +// BootstrapNew creates a new snapshotter and WAL writer, expecting that there is nothing on disk +func (e *EncryptedRaftLogger) BootstrapNew(metadata []byte) error { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + encrypter, decrypter := encryption.Defaults(e.EncryptionKey) + walFactory := NewWALFactory(encrypter, decrypter) + + for _, dirpath := range []string{e.walDir(), e.snapDir()} { + if err := os.MkdirAll(dirpath, 0700); err != nil { + return errors.Wrapf(err, "failed to create %s", dirpath) + } + } + var err error + e.wal, err = walFactory.Create(e.walDir(), metadata) + if err != nil { + return errors.Wrap(err, "failed to create WAL") + } + + e.snapshotter = NewSnapFactory(encrypter, decrypter).New(e.snapDir()) + return nil +} + +func (e *EncryptedRaftLogger) walDir() string { + return filepath.Join(e.StateDir, versionedWALSnapDirs[0].wal) +} + +func (e *EncryptedRaftLogger) snapDir() string { + return filepath.Join(e.StateDir, versionedWALSnapDirs[0].snap) +} + +// RotateEncryptionKey swaps out the encoders and decoders used by the wal and snapshotter +func (e *EncryptedRaftLogger) RotateEncryptionKey(newKey []byte) { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + if e.wal != nil { // if the wal exists, the snapshotter exists + // We don't want to have to close the WAL, because we can't open a new one. + // We need to know the previous snapshot, because when you open a WAL you + // have to read out all the entries from a particular snapshot, or you can't + // write. So just rotate the encoders out from under it. We already + // have a lock on writing to snapshots and WALs. + wrapped, ok := e.wal.(*wrappedWAL) + if !ok { + panic(fmt.Errorf("EncryptedRaftLogger's WAL is not a wrappedWAL")) + } + + wrapped.encrypter, wrapped.decrypter = encryption.Defaults(newKey) + + e.snapshotter = NewSnapFactory(wrapped.encrypter, wrapped.decrypter).New(e.snapDir()) + } + e.EncryptionKey = newKey +} + +// SaveSnapshot actually saves a given snapshot to both the WAL and the snapshot. +func (e *EncryptedRaftLogger) SaveSnapshot(snapshot raftpb.Snapshot) error { + + walsnap := walpb.Snapshot{ + Index: snapshot.Metadata.Index, + Term: snapshot.Metadata.Term, + } + + e.encoderMu.RLock() + if err := e.wal.SaveSnapshot(walsnap); err != nil { + e.encoderMu.RUnlock() + return err + } + + snapshotter := e.snapshotter + e.encoderMu.RUnlock() + + if err := snapshotter.SaveSnap(snapshot); err != nil { + return err + } + if err := e.wal.ReleaseLockTo(snapshot.Metadata.Index); err != nil { + return err + } + return nil +} + +// GC garbage collects snapshots and wals older than the provided index and term +func (e *EncryptedRaftLogger) GC(index uint64, term uint64, keepOldSnapshots uint64) error { + // Delete any older snapshots + curSnapshot := fmt.Sprintf("%016x-%016x%s", term, index, ".snap") + + snapshots, err := ListSnapshots(e.snapDir()) + if err != nil { + return err + } + + // Ignore any snapshots that are older than the current snapshot. + // Delete the others. Rather than doing lexical comparisons, we look + // at what exists before/after the current snapshot in the slice. + // This means that if the current snapshot doesn't appear in the + // directory for some strange reason, we won't delete anything, which + // is the safe behavior. + curSnapshotIdx := -1 + var ( + removeErr error + oldestSnapshot string + ) + + for i, snapFile := range snapshots { + if curSnapshotIdx >= 0 && i > curSnapshotIdx { + if uint64(i-curSnapshotIdx) > keepOldSnapshots { + err := os.Remove(filepath.Join(e.snapDir(), snapFile)) + if err != nil && removeErr == nil { + removeErr = err + } + continue + } + } else if snapFile == curSnapshot { + curSnapshotIdx = i + } + oldestSnapshot = snapFile + } + + if removeErr != nil { + return removeErr + } + + // Remove any WAL files that only contain data from before the oldest + // remaining snapshot. + + if oldestSnapshot == "" { + return nil + } + + // Parse index out of oldest snapshot's filename + var snapTerm, snapIndex uint64 + _, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex) + if err != nil { + return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot) + } + + wals, err := ListWALs(e.walDir()) + if err != nil { + return err + } + + found := false + deleteUntil := -1 + + for i, walName := range wals { + var walSeq, walIndex uint64 + _, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex) + if err != nil { + return errors.Wrapf(err, "could not parse WAL name %s", walName) + } + + if walIndex >= snapIndex { + deleteUntil = i - 1 + found = true + break + } + } + + // If all WAL files started with indices below the oldest snapshot's + // index, we can delete all but the newest WAL file. + if !found && len(wals) != 0 { + deleteUntil = len(wals) - 1 + } + + for i := 0; i < deleteUntil; i++ { + walPath := filepath.Join(e.walDir(), wals[i]) + l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode) + if err != nil { + return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i]) + } + err = os.Remove(walPath) + l.Close() + if err != nil { + return errors.Wrapf(err, "error removing old WAL file %s", wals[i]) + } + } + + return nil +} + +// SaveEntries saves only entries to disk +func (e *EncryptedRaftLogger) SaveEntries(st raftpb.HardState, entries []raftpb.Entry) error { + e.encoderMu.RLock() + defer e.encoderMu.RUnlock() + + if e.wal == nil { + return fmt.Errorf("raft WAL has either been closed or has never been created") + } + return e.wal.Save(st, entries) +} + +// Close closes the logger - it will have to be bootstrapped again to start writing +func (e *EncryptedRaftLogger) Close(ctx context.Context) { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + if e.wal != nil { + if err := e.wal.Close(); err != nil { + log.G(ctx).WithError(err).Error("error closing raft WAL") + } + } + + e.wal = nil + e.snapshotter = nil +} + +// Clear closes the existing WAL and moves away the WAL and snapshot. +func (e *EncryptedRaftLogger) Clear(ctx context.Context) error { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + if e.wal != nil { + if err := e.wal.Close(); err != nil { + log.G(ctx).WithError(err).Error("error closing raft WAL") + } + } + e.snapshotter = nil + + newWALDir, err := ioutil.TempDir(e.StateDir, "wal.") + if err != nil { + return err + } + err = os.Rename(e.walDir(), newWALDir) + if err != nil { + return err + } + + newSnapDir, err := ioutil.TempDir(e.StateDir, "snap.") + if err != nil { + return err + } + err = os.Rename(e.snapDir(), newSnapDir) + if err != nil { + return err + } + + return nil +} diff --git a/manager/state/raft/storage/storage_test.go b/manager/state/raft/storage/storage_test.go new file mode 100644 index 0000000000..2683d1d735 --- /dev/null +++ b/manager/state/raft/storage/storage_test.go @@ -0,0 +1,221 @@ +package storage + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/coreos/etcd/raft/raftpb" + "github.com/coreos/etcd/wal/walpb" + "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestBootstrapFromDisk(t *testing.T) { + tempdir, err := ioutil.TempDir("", "raft-storage") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key1"), + } + err = logger.BootstrapNew([]byte("metadata")) + require.NoError(t, err) + + // everything should be saved with "key1" + _, entries, _ := makeWALData(0, 0) + err = logger.SaveEntries(raftpb.HardState{}, entries) + require.NoError(t, err) + logger.Close(context.Background()) + + // now we can bootstrap from disk, even if there is no snapshot + logger = EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key1"), + } + readSnap, waldata, err := logger.BootstrapFromDisk(context.Background()) + require.NoError(t, err) + require.Nil(t, readSnap) + require.Equal(t, entries, waldata.Entries) + + // save a snapshot + snapshot := fakeSnapshotData + err = logger.SaveSnapshot(snapshot) + require.NoError(t, err) + _, entries, _ = makeWALData(snapshot.Metadata.Index, snapshot.Metadata.Term) + err = logger.SaveEntries(raftpb.HardState{}, entries) + require.NoError(t, err) + logger.Close(context.Background()) + + // load snapshots and wals + logger = EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key1"), + } + readSnap, waldata, err = logger.BootstrapFromDisk(context.Background()) + require.NoError(t, err) + require.NotNil(t, snapshot) + require.Equal(t, snapshot, *readSnap) + require.Equal(t, entries, waldata.Entries) + + // start writing more wals and rotate in the middle + _, entries, _ = makeWALData(snapshot.Metadata.Index, snapshot.Metadata.Term) + err = logger.SaveEntries(raftpb.HardState{}, entries[:1]) + require.NoError(t, err) + logger.RotateEncryptionKey([]byte("key2")) + err = logger.SaveEntries(raftpb.HardState{}, entries[1:]) + require.NoError(t, err) + logger.Close(context.Background()) + + // we can't bootstrap from disk using only the first or second key + for _, key := range []string{"key1", "key2"} { + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte(key), + } + _, _, err := logger.BootstrapFromDisk(context.Background()) + require.IsType(t, encryption.ErrCannotDecrypt{}, errors.Cause(err)) + } + + // but we can if we combine the two keys, we can bootstrap just fine + logger = EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key2"), + } + readSnap, waldata, err = logger.BootstrapFromDisk(context.Background(), []byte("key1")) + require.NoError(t, err) + require.NotNil(t, snapshot) + require.Equal(t, snapshot, *readSnap) + require.Equal(t, entries, waldata.Entries) +} + +// Ensure that we can change encoding and not have a race condition +func TestRaftLoggerRace(t *testing.T) { + tempdir, err := ioutil.TempDir("", "raft-storage") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("Hello"), + } + err = logger.BootstrapNew([]byte("metadata")) + require.NoError(t, err) + + _, entries, _ := makeWALData(fakeSnapshotData.Metadata.Index, fakeSnapshotData.Metadata.Term) + + done1 := make(chan error) + done2 := make(chan error) + done3 := make(chan error) + done4 := make(chan error) + go func() { + done1 <- logger.SaveSnapshot(fakeSnapshotData) + }() + go func() { + done2 <- logger.SaveEntries(raftpb.HardState{}, entries) + }() + go func() { + logger.RotateEncryptionKey([]byte("Hello 2")) + done3 <- nil + }() + go func() { + done4 <- logger.SaveSnapshot(fakeSnapshotData) + }() + + err = <-done1 + require.NoError(t, err, "unable to save snapshot") + + err = <-done2 + require.NoError(t, err, "unable to save entries") + + err = <-done3 + require.NoError(t, err, "unable to rotate key") + + err = <-done4 + require.NoError(t, err, "unable to save snapshot a second time") +} + +func TestMigrateToV3EncryptedForm(t *testing.T) { + t.Parallel() + + tempdir, err := ioutil.TempDir("", "raft-storage") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + dek := []byte("key") + + writeDataTo := func(suffix string, snapshot raftpb.Snapshot, walFactory WALFactory, snapFactory SnapFactory) []raftpb.Entry { + snapDir := filepath.Join(tempdir, "snap"+suffix) + walDir := filepath.Join(tempdir, "wal"+suffix) + for _, dirpath := range []string{snapDir, walDir} { + require.NoError(t, os.MkdirAll(dirpath, 0755)) + } + require.NoError(t, snapFactory.New(snapDir).SaveSnap(snapshot)) + + _, entries, _ := makeWALData(snapshot.Metadata.Index, snapshot.Metadata.Term) + walWriter, err := walFactory.Create(walDir, []byte("metadata")) + require.NoError(t, err) + require.NoError(t, walWriter.SaveSnapshot(walpb.Snapshot{Index: snapshot.Metadata.Index, Term: snapshot.Metadata.Term})) + require.NoError(t, walWriter.Save(raftpb.HardState{}, entries)) + require.NoError(t, walWriter.Close()) + return entries + } + + requireLoadedData := func(expectedSnap raftpb.Snapshot, expectedEntries []raftpb.Entry) { + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: dek, + } + readSnap, waldata, err := logger.BootstrapFromDisk(context.Background()) + require.NoError(t, err) + require.NotNil(t, readSnap) + require.Equal(t, expectedSnap, *readSnap) + require.Equal(t, expectedEntries, waldata.Entries) + logger.Close(context.Background()) + } + + v2Snapshot := fakeSnapshotData + v3Snapshot := fakeSnapshotData + v3Snapshot.Metadata.Index += 100 + v3Snapshot.Metadata.Term += 10 + v3EncryptedSnapshot := fakeSnapshotData + v3EncryptedSnapshot.Metadata.Index += 200 + v3EncryptedSnapshot.Metadata.Term += 20 + + encoder, decoders := encryption.Defaults(dek) + walFactory := NewWALFactory(encoder, decoders) + snapFactory := NewSnapFactory(encoder, decoders) + + // generate both v2 and v3 unencrypted snapshot data directories, as well as an encrypted directory + v2Entries := writeDataTo("", v2Snapshot, OriginalWAL, OriginalSnap) + v3Entries := writeDataTo("-v3", v3Snapshot, OriginalWAL, OriginalSnap) + v3EncryptedEntries := writeDataTo("-v3-encrypted", v3EncryptedSnapshot, walFactory, snapFactory) + + // bootstrap from disk - the encrypted directory exists, so we should just read from + // it instead of from the legacy directories + requireLoadedData(v3EncryptedSnapshot, v3EncryptedEntries) + + // remove the newest dirs - should try to migrate from v3 + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3-encrypted"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "wal-v3-encrypted"))) + requireLoadedData(v3Snapshot, v3Entries) + // it can recover from partial migrations + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3-encrypted"))) + requireLoadedData(v3Snapshot, v3Entries) + // v3 dirs still there + _, err = os.Stat(filepath.Join(tempdir, "snap-v3")) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(tempdir, "wal-v3")) + require.NoError(t, err) + + // remove the v3 dirs - should try to migrate from v2 + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3-encrypted"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "wal-v3-encrypted"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "wal-v3"))) + requireLoadedData(v2Snapshot, v2Entries) +} diff --git a/manager/state/raft/storage/walwrap.go b/manager/state/raft/storage/walwrap.go index fdd663262b..87009d4827 100644 --- a/manager/state/raft/storage/walwrap.go +++ b/manager/state/raft/storage/walwrap.go @@ -1,10 +1,21 @@ package storage import ( + "context" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/raft/raftpb" "github.com/coreos/etcd/wal" "github.com/coreos/etcd/wal/walpb" + "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" ) // This package wraps the github.com/coreos/etcd/wal package, and encrypts @@ -128,3 +139,115 @@ func (o originalWAL) Open(dirpath string, walsnap walpb.Snapshot) (WAL, error) { // OriginalWAL is the original `wal` package as an implemntation of the WALFactory interface var OriginalWAL WALFactory = originalWAL{} + +// WALData contains all the data returned by a WAL's ReadAll() function +// (metadata, hardwate, and entries) +type WALData struct { + Metadata []byte + HardState raftpb.HardState + Entries []raftpb.Entry +} + +// ReadRepairWAL opens a WAL for reading, and attempts to read it. If we can't read it, attempts to repair +// and read again. +func ReadRepairWAL( + ctx context.Context, + walDir string, + walsnap walpb.Snapshot, + factory WALFactory, +) (WAL, WALData, error) { + var ( + reader WAL + metadata []byte + st raftpb.HardState + ents []raftpb.Entry + err error + ) + repaired := false + for { + if reader, err = factory.Open(walDir, walsnap); err != nil { + return nil, WALData{}, errors.Wrap(err, "failed to open WAL") + } + if metadata, st, ents, err = reader.ReadAll(); err != nil { + if closeErr := reader.Close(); closeErr != nil { + return nil, WALData{}, closeErr + } + if _, ok := err.(encryption.ErrCannotDecrypt); ok { + return nil, WALData{}, errors.Wrap(err, "failed to decrypt WAL") + } + // we can only repair ErrUnexpectedEOF and we never repair twice. + if repaired || err != io.ErrUnexpectedEOF { + return nil, WALData{}, errors.Wrap(err, "irreparable WAL error") + } + if !wal.Repair(walDir) { + return nil, WALData{}, errors.Wrap(err, "WAL error cannot be repaired") + } + log.G(ctx).WithError(err).Info("repaired WAL error") + repaired = true + continue + } + break + } + return reader, WALData{ + Metadata: metadata, + HardState: st, + Entries: ents, + }, nil +} + +// MigrateWALs reads existing WALs (from a particular snapshot and beyond) from one directory, encoded one way, +// and writes them to a new directory, encoded a different way +func MigrateWALs(ctx context.Context, oldDir, newDir string, oldFactory, newFactory WALFactory, snapshot walpb.Snapshot) error { + oldReader, waldata, err := ReadRepairWAL(ctx, oldDir, snapshot, oldFactory) + if err != nil { + return err + } + oldReader.Close() + + // keep temporary wal directory so WAL initialization appears atomic + tmpdirpath := filepath.Clean(newDir) + ".tmp" + if fileutil.Exist(tmpdirpath) { + if err := os.RemoveAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not remove temporary WAL directory") + } + } + if err := fileutil.CreateDirAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not create temporary WAL directory") + } + + tmpWAL, err := newFactory.Create(tmpdirpath, waldata.Metadata) + if err != nil { + return errors.Wrap(err, "could not create new WAL in temporary WAL directory") + } + defer tmpWAL.Close() + + if err := tmpWAL.SaveSnapshot(snapshot); err != nil { + return errors.Wrap(err, "could not write WAL snapshot in temporary directory") + } + + if err := tmpWAL.Save(waldata.HardState, waldata.Entries); err != nil { + return errors.Wrap(err, "could not migrate WALs to temporary directory") + } + + return os.Rename(tmpdirpath, newDir) +} + +// ListWALs lists all the wals in a directory and returns the list in lexical +// order (oldest first) +func ListWALs(dirpath string) ([]string, error) { + dirents, err := ioutil.ReadDir(dirpath) + if err != nil { + return nil, err + } + + var wals []string + for _, dirent := range dirents { + if strings.HasSuffix(dirent.Name(), ".wal") { + wals = append(wals, dirent.Name()) + } + } + + // Sort WAL filenames in lexical order + sort.Sort(sort.StringSlice(wals)) + return wals, nil +} diff --git a/manager/state/raft/storage/walwrap_test.go b/manager/state/raft/storage/walwrap_test.go index 6f93f22f94..24ca563eb5 100644 --- a/manager/state/raft/storage/walwrap_test.go +++ b/manager/state/raft/storage/walwrap_test.go @@ -2,9 +2,11 @@ package storage import ( "bytes" + "context" "fmt" "io/ioutil" "os" + "path/filepath" "testing" "github.com/coreos/etcd/raft/raftpb" @@ -17,20 +19,22 @@ import ( var _ WALFactory = walCryptor{} // Generates a bunch of WAL test data -func makeWALData() ([]byte, []raftpb.Entry, walpb.Snapshot) { - term := uint64(3) - index := uint64(4) +func makeWALData(index uint64, term uint64) ([]byte, []raftpb.Entry, walpb.Snapshot) { + wsn := walpb.Snapshot{ + Index: index, + Term: term, + } var entries []raftpb.Entry - for i := index + 1; i < index+6; i++ { + for i := wsn.Index + 1; i < wsn.Index+6; i++ { entries = append(entries, raftpb.Entry{ - Term: term, + Term: wsn.Term + 1, Index: i, Data: []byte(fmt.Sprintf("Entry %d", i)), }) } - return []byte("metadata"), entries, walpb.Snapshot{Index: index, Term: term} + return []byte("metadata"), entries, wsn } func createWithWAL(t *testing.T, w WALFactory, metadata []byte, startSnap walpb.Snapshot, entries []raftpb.Entry) string { @@ -49,7 +53,7 @@ func createWithWAL(t *testing.T, w WALFactory, metadata []byte, startSnap walpb. // WAL can read entries are not wrapped, but not encrypted func TestReadAllWrappedNoEncryption(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) wrappedEntries := make([]raftpb.Entry, len(entries)) for i, entry := range entries { r := api.MaybeEncryptedRecord{Data: entry.Data} @@ -77,7 +81,7 @@ func TestReadAllWrappedNoEncryption(t *testing.T) { // When reading WAL, if the decrypter can't read the encryption type, errors func TestReadAllNoSupportedDecrypter(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) for i, entry := range entries { r := api.MaybeEncryptedRecord{Data: entry.Data, Algorithm: api.MaybeEncryptedRecord_Algorithm(-3)} data, err := r.Marshal() @@ -102,7 +106,7 @@ func TestReadAllNoSupportedDecrypter(t *testing.T) { // entry is incorrectly encryptd, an error is returned func TestReadAllEntryIncorrectlyEncrypted(t *testing.T) { crypter := &meowCrypter{} - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) // metadata is correctly encryptd, but entries are not meow-encryptd for i, entry := range entries { @@ -128,7 +132,7 @@ func TestReadAllEntryIncorrectlyEncrypted(t *testing.T) { // The entry data and metadata are encryptd with the given encrypter, and a regular // WAL will see them as such. func TestSave(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) crypter := &meowCrypter{} c := NewWALFactory(crypter, encryption.NoopCrypter) @@ -154,7 +158,7 @@ func TestSave(t *testing.T) { // If encryption fails, saving will fail func TestSaveEncryptionFails(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) tempdir, err := ioutil.TempDir("", "waltests") require.NoError(t, err) @@ -162,7 +166,7 @@ func TestSaveEncryptionFails(t *testing.T) { // fail encrypting one of the entries, but not the first one c := NewWALFactory(&meowCrypter{encryptFailures: map[string]struct{}{ - "Entry 7": {}, + "Entry 3": {}, }}, nil) wrapped, err := c.Create(tempdir, metadata) require.NoError(t, err) @@ -198,7 +202,7 @@ func TestCreateOpenInvalidDirFails(t *testing.T) { // A WAL can read what it wrote so long as it has a corresponding decrypter func TestSaveAndRead(t *testing.T) { crypter := &meowCrypter{} - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) c := NewWALFactory(crypter, crypter) tempdir := createWithWAL(t, c, metadata, snapshot, entries) @@ -213,3 +217,97 @@ func TestSaveAndRead(t *testing.T) { require.Equal(t, metadata, meta) require.Equal(t, entries, ents) } + +func TestReadRepairWAL(t *testing.T) { + metadata, entries, snapshot := makeWALData(1, 1) + tempdir := createWithWAL(t, OriginalWAL, metadata, snapshot, entries) + defer os.RemoveAll(tempdir) + + // there should only be one WAL file in there - corrupt it + files, err := ioutil.ReadDir(tempdir) + require.NoError(t, err) + require.Len(t, files, 1) + + fName := filepath.Join(tempdir, files[0].Name()) + fileContents, err := ioutil.ReadFile(fName) + require.NoError(t, err) + require.NoError(t, ioutil.WriteFile(fName, fileContents[:200], files[0].Mode())) + + ogWAL, err := OriginalWAL.Open(tempdir, snapshot) + require.NoError(t, err) + _, _, _, err = ogWAL.ReadAll() + require.Error(t, err) + require.NoError(t, ogWAL.Close()) + + ogWAL, waldata, err := ReadRepairWAL(context.Background(), tempdir, snapshot, OriginalWAL) + require.NoError(t, err) + require.Equal(t, metadata, waldata.Metadata) + require.NoError(t, ogWAL.Close()) +} + +func TestMigrateWALs(t *testing.T) { + metadata, entries, snapshot := makeWALData(1, 1) + coder := &meowCrypter{} + c := NewWALFactory(coder, coder) + + var ( + err error + dirs = make([]string, 2) + ) + + tempDir, err := ioutil.TempDir("", "test-migrate") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + for i := range dirs { + dirs[i] = filepath.Join(tempDir, fmt.Sprintf("walDir%d", i)) + } + + origDir := createWithWAL(t, OriginalWAL, metadata, snapshot, entries) + defer os.RemoveAll(origDir) + + // original to new + oldDir := origDir + newDir := dirs[0] + + err = MigrateWALs(context.Background(), oldDir, newDir, OriginalWAL, c, snapshot) + require.NoError(t, err) + + newWAL, err := c.Open(newDir, snapshot) + require.NoError(t, err) + meta, _, ents, err := newWAL.ReadAll() + require.NoError(t, err) + require.Equal(t, metadata, meta) + require.Equal(t, entries, ents) + require.NoError(t, newWAL.Close()) + + // new to original + oldDir = dirs[0] + newDir = dirs[1] + + err = MigrateWALs(context.Background(), oldDir, newDir, c, OriginalWAL, snapshot) + require.NoError(t, err) + + newWAL, err = OriginalWAL.Open(newDir, snapshot) + require.NoError(t, err) + meta, _, ents, err = newWAL.ReadAll() + require.NoError(t, err) + require.Equal(t, metadata, meta) + require.Equal(t, entries, ents) + require.NoError(t, newWAL.Close()) + + // If we can't read the old directory (for instance if it doesn't exist), a temp directory + // is not created + for _, dir := range dirs { + require.NoError(t, os.RemoveAll(dir)) + } + oldDir = dirs[0] + newDir = dirs[1] + + err = MigrateWALs(context.Background(), oldDir, newDir, OriginalWAL, c, walpb.Snapshot{}) + require.Error(t, err) + + subdirs, err := ioutil.ReadDir(tempDir) + require.NoError(t, err) + require.Empty(t, subdirs) +} diff --git a/manager/state/raft/storage_test.go b/manager/state/raft/storage_test.go index ba8af148cd..61052a2392 100644 --- a/manager/state/raft/storage_test.go +++ b/manager/state/raft/storage_test.go @@ -10,6 +10,8 @@ import ( "time" "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/manager/state/raft" + "github.com/docker/swarmkit/manager/state/raft/storage" raftutils "github.com/docker/swarmkit/manager/state/raft/testutils" "github.com/docker/swarmkit/manager/state/store" "github.com/pivotal-golang/clock/fakeclock" @@ -38,7 +40,7 @@ func TestRaftSnapshot(t *testing.T) { // None of the nodes should have snapshot files yet for _, node := range nodes { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) assert.NoError(t, err) assert.Len(t, dirents, 0) } @@ -57,7 +59,7 @@ func TestRaftSnapshot(t *testing.T) { // All nodes should now have a snapshot file for nodeID, node := range nodes { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -74,7 +76,7 @@ func TestRaftSnapshot(t *testing.T) { // It should get a copy of the snapshot assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[4].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[4].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -110,7 +112,7 @@ func TestRaftSnapshot(t *testing.T) { // All nodes should have a snapshot under a *different* name for nodeID, node := range nodes { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -118,7 +120,7 @@ func TestRaftSnapshot(t *testing.T) { return fmt.Errorf("expected 1 snapshot, found %d on node %d", len(dirents), nodeID) } if dirents[0].Name() == snapshotFilenames[nodeID] { - return fmt.Errorf("snapshot %s did not get replaced", snapshotFilenames[nodeID]) + return fmt.Errorf("snapshot %s did not get replaced on node %d", snapshotFilenames[nodeID], nodeID) } return nil })) @@ -155,7 +157,7 @@ func TestRaftSnapshotRestart(t *testing.T) { // Remaining nodes shouldn't have snapshot files yet for _, node := range []*raftutils.TestNode{nodes[1], nodes[2]} { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) assert.NoError(t, err) assert.Len(t, dirents, 0) } @@ -168,7 +170,7 @@ func TestRaftSnapshotRestart(t *testing.T) { // Remaining nodes should now have a snapshot file for nodeIdx, node := range []*raftutils.TestNode{nodes[1], nodes[2]} { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -190,7 +192,7 @@ func TestRaftSnapshotRestart(t *testing.T) { // New node should get a copy of the snapshot assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -200,7 +202,7 @@ func TestRaftSnapshotRestart(t *testing.T) { return nil })) - dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3-encrypted")) assert.NoError(t, err) assert.Len(t, dirents, 1) raftutils.CheckValuesOnNodes(t, clockSource, map[uint64]*raftutils.TestNode{1: nodes[1], 2: nodes[2]}, nodeIDs[:5], values[:5]) @@ -271,7 +273,7 @@ func TestGCWAL(t *testing.T) { // Snapshot should have been triggered just as the WAL rotated, so // both WAL files should be preserved assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -279,7 +281,7 @@ func TestGCWAL(t *testing.T) { return fmt.Errorf("expected 1 snapshot, found %d", len(dirents)) } - dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3")) + dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted")) if err != nil { return err } @@ -311,7 +313,7 @@ func TestGCWAL(t *testing.T) { // This time only one WAL file should be saved. assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -320,7 +322,7 @@ func TestGCWAL(t *testing.T) { return fmt.Errorf("expected 1 snapshot, found %d", len(dirents)) } - dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3")) + dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted")) if err != nil { return err } @@ -437,23 +439,229 @@ func proposeLargeValue(t *testing.T, raftNode *raftutils.TestNode, time time.Dur return node, nil } -func TestMigrateWAL(t *testing.T) { +func TestRaftEncryptionKeyRotation(t *testing.T) { + t.Parallel() nodes := make(map[uint64]*raftutils.TestNode) var clockSource *fakeclock.FakeClock - nodes[1], clockSource = raftutils.NewInitNode(t, tc, nil) + raftConfig := raft.DefaultRaftConfig() + nodes[1], clockSource = raftutils.NewInitNode(t, tc, &raftConfig) defer raftutils.TeardownCluster(t, nodes) - value, err := raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, "id1") - assert.NoError(t, err, "failed to propose value") - raftutils.CheckValuesOnNodes(t, clockSource, nodes, []string{"id1"}, []*api.Node{value}) + nodeIDs := []string{"id1", "id2", "id3", "id4", "id5", "id6", "id7"} + values := make([]*api.Node, len(nodeIDs)) + + // Propose 3 values + var err error + for i, nodeID := range nodeIDs[:3] { + values[i], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeID) + require.NoError(t, err, "failed to propose value") + } + + snapDir := filepath.Join(nodes[1].StateDir, "snap-v3-encrypted") + + startingKeys := nodes[1].KeyRotator.GetKeys() + + // rotate the encryption key + nodes[1].KeyRotator.QueuePendingKey([]byte("key2")) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + // the rotation should trigger a snapshot, which should notify the rotator when it's done + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshot, found %d on new node", len(snapshots)) + } + if nodes[1].KeyRotator.NeedsRotation() { + return fmt.Errorf("rotation never finished") + } + return nil + })) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:3], values[:3]) + + // Propose a 4th value + values[3], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[3]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:4], values[:4]) nodes[1].Server.Stop() nodes[1].ShutdownRaft() - // Move WAL directory so it looks like it was created by an old version - require.NoError(t, os.Rename(filepath.Join(nodes[1].StateDir, "wal-v3"), filepath.Join(nodes[1].StateDir, "wal"))) + // Try to restart node 1. Without the new unlock key, it can't actually start + n, ctx := raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(startingKeys)) + require.Error(t, n.Node.JoinAndStart(ctx), + "should not have been able to restart since we can't read snapshots") + // with the right key, it can start, even if the right key is only the pending key + newKeys := startingKeys + newKeys.PendingDEK = []byte("key2") + nodes[1].KeyRotator = raftutils.NewSimpleKeyRotator(newKeys) nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], false) - raftutils.CheckValuesOnNodes(t, clockSource, nodes, []string{"id1"}, []*api.Node{value}) + + raftutils.WaitForCluster(t, clockSource, nodes) + + // as soon as we joined, it should have finished rotating the key + require.False(t, nodes[1].KeyRotator.NeedsRotation()) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:4], values[:4]) + + // break snapshotting, and ensure that key rotation never finishes + tempSnapDir := filepath.Join(nodes[1].StateDir, "snap-backup") + require.NoError(t, os.Rename(snapDir, tempSnapDir)) + require.NoError(t, ioutil.WriteFile(snapDir, []byte("this is no longer a directory"), 0644)) + + nodes[1].KeyRotator.QueuePendingKey([]byte("key3")) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + time.Sleep(250 * time.Millisecond) + + // rotation has not been finished, because we cannot take a snapshot + require.True(t, nodes[1].KeyRotator.NeedsRotation()) + + // Propose a 5th value, so we have WALs written with the new key + values[4], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[4]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:5], values[:5]) + + nodes[1].Server.Stop() + nodes[1].ShutdownRaft() + + // restore the snapshot dir + require.NoError(t, os.RemoveAll(snapDir)) + require.NoError(t, os.Rename(tempSnapDir, snapDir)) + + // Now the wals are a mix of key2 and key3 - we can't actually start with either key + singleKey := raft.EncryptionKeys{CurrentDEK: []byte("key2")} + n, ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(singleKey)) + require.Error(t, n.Node.JoinAndStart(ctx), + "should not have been able to restart since we can't read all the WALs, even if we can read the snapshot") + singleKey = raft.EncryptionKeys{CurrentDEK: []byte("key3")} + n, ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(singleKey)) + require.Error(t, n.Node.JoinAndStart(ctx), + "should not have been able to restart since we can't read all the WALs, and also not the snapshot") + + nodes[1], ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, + raftutils.NewSimpleKeyRotator(raft.EncryptionKeys{ + CurrentDEK: []byte("key2"), + PendingDEK: []byte("key3"), + })) + require.NoError(t, nodes[1].Node.JoinAndStart(ctx)) + + // we can load, but we still need a snapshot because rotation hasn't finished + snapshots, err := storage.ListSnapshots(snapDir) + require.NoError(t, err) + require.Len(t, snapshots, 1, "expected 1 snapshot") + require.True(t, nodes[1].KeyRotator.NeedsRotation()) + currSnapshot := snapshots[0] + + // start the node - everything should fix itself + go nodes[1].Node.Run(ctx) + raftutils.WaitForCluster(t, clockSource, nodes) + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + if nodes[1].KeyRotator.NeedsRotation() { + return fmt.Errorf("rotation never finished") + } + currSnapshot = snapshots[0] + return nil + })) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:5], values[:5]) + + // If we can't update the keys, we wait for the next snapshot to do so + nodes[1].KeyRotator.SetUpdateFunc(func() error { return fmt.Errorf("nope!") }) + nodes[1].KeyRotator.QueuePendingKey([]byte("key4")) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + currSnapshot = snapshots[0] + return nil + })) + require.True(t, nodes[1].KeyRotator.NeedsRotation()) + + // Fix updating the key rotator, and propose a 6th value, so another snapshot is + // triggered. + nodes[1].KeyRotator.SetUpdateFunc(nil) + values[5], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[5]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:6], values[:6]) + raftutils.AdvanceTicks(clockSource, 5) + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + if nodes[1].KeyRotator.NeedsRotation() { + return fmt.Errorf("rotation never finished") + } + currSnapshot = snapshots[0] + return nil + })) + require.False(t, nodes[1].KeyRotator.NeedsRotation()) + + // Even if something goes wrong with getting keys, and needs rotation returns a false positive, + // if there's no PendingDEK nothing happens. A snapshot is spuriously triggered, but it writes + // with the current DEK. + + // propose another value, so snapshotting doesn't fail + values[6], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[6]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:7], values[:7]) + + fakeTrue := true + nodes[1].KeyRotator.SetNeedsRotation(&fakeTrue) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + return nil + })) + + nodes[1].Server.Stop() + nodes[1].ShutdownRaft() + + // We can decrypt with key4 - no key5 + nodes[1].KeyRotator = raftutils.NewSimpleKeyRotator(raft.EncryptionKeys{ + CurrentDEK: []byte("key4"), + }) + nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], false) + raftutils.WaitForCluster(t, clockSource, nodes) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:7], values[:7]) } diff --git a/manager/state/raft/testutils/testutils.go b/manager/state/raft/testutils/testutils.go index 4c92d1c3c0..2c34b44d23 100644 --- a/manager/state/raft/testutils/testutils.go +++ b/manager/state/raft/testutils/testutils.go @@ -5,6 +5,7 @@ import ( "net" "os" "reflect" + "sync" "testing" "time" @@ -35,6 +36,7 @@ type TestNode struct { Address string StateDir string cancel context.CancelFunc + KeyRotator *SimpleKeyRotator } // Leader is wrapper around real Leader method to suppress error. @@ -201,6 +203,79 @@ func RecycleWrappedListener(old *WrappedListener) *WrappedListener { } } +// SimpleKeyRotator does some DEK rotation +type SimpleKeyRotator struct { + mu sync.Mutex + rotateCh chan struct{} + updateFunc func() error + overrideNeedRotate *bool + raft.EncryptionKeys +} + +// GetKeys returns the current set of keys +func (s *SimpleKeyRotator) GetKeys() raft.EncryptionKeys { + s.mu.Lock() + defer s.mu.Unlock() + return s.EncryptionKeys +} + +// NeedsRotation returns whether we need to rotate +func (s *SimpleKeyRotator) NeedsRotation() bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.overrideNeedRotate != nil { + return *s.overrideNeedRotate + } + return s.EncryptionKeys.PendingDEK != nil +} + +// UpdateKeys updates the current encryption keys +func (s *SimpleKeyRotator) UpdateKeys(newKeys raft.EncryptionKeys) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.updateFunc != nil { + return s.updateFunc() + } + s.EncryptionKeys = newKeys + return nil +} + +// RotationNotify returns the rotation notification channel +func (s *SimpleKeyRotator) RotationNotify() chan struct{} { + return s.rotateCh +} + +// QueuePendingKey lets us rotate the key +func (s *SimpleKeyRotator) QueuePendingKey(key []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.EncryptionKeys.PendingDEK = key +} + +// SetUpdateFunc enables you to inject an error when updating keys +func (s *SimpleKeyRotator) SetUpdateFunc(updateFunc func() error) { + s.mu.Lock() + defer s.mu.Unlock() + s.updateFunc = updateFunc +} + +// SetNeedsRotation enables you to inject a value for NeedsRotation +func (s *SimpleKeyRotator) SetNeedsRotation(override *bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.overrideNeedRotate = override +} + +// NewSimpleKeyRotator returns a basic EncryptionKeyRotator +func NewSimpleKeyRotator(keys raft.EncryptionKeys) *SimpleKeyRotator { + return &SimpleKeyRotator{ + rotateCh: make(chan struct{}), + EncryptionKeys: keys, + } +} + +var _ raft.EncryptionKeyRotator = NewSimpleKeyRotator(raft.EncryptionKeys{}) + // NewNode creates a new raft node to use for tests func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, opts ...raft.NodeOptions) *TestNode { l, err := net.Listen("tcp", "127.0.0.1:0") @@ -218,6 +293,7 @@ func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, stateDir, err := ioutil.TempDir("", "test-raft") require.NoError(t, err, "can't create temporary state directory") + keyRotator := NewSimpleKeyRotator(raft.EncryptionKeys{CurrentDEK: []byte("current")}) newNodeOpts := raft.NodeOptions{ ID: securityConfig.ClientTLSCreds.NodeID(), Addr: l.Addr().String(), @@ -226,6 +302,7 @@ func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, ClockSource: clockSource, SendTimeout: 10 * time.Second, TLSCredentials: securityConfig.ClientTLSCreds, + KeyRotator: keyRotator, } if len(opts) > 1 { @@ -258,6 +335,7 @@ func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, Address: newNodeOpts.Addr, StateDir: newNodeOpts.StateDir, Server: s, + KeyRotator: keyRotator, } } @@ -316,8 +394,8 @@ func NewJoinNode(t *testing.T, clockSource *fakeclock.FakeClock, join string, tc return n } -// RestartNode restarts a raft test node -func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNode, forceNewCluster bool) *TestNode { +// CopyNode returns a copy of a node +func CopyNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNode, forceNewCluster bool, kr *SimpleKeyRotator) (*TestNode, context.Context) { wrappedListener := RecycleWrappedListener(oldNode.Listener) securityConfig := oldNode.SecurityConfig serverOpts := []grpc.ServerOption{grpc.Creds(securityConfig.ServerTLSCreds)} @@ -325,6 +403,10 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo cfg := raft.DefaultNodeConfig() + if kr == nil { + kr = oldNode.KeyRotator + } + newNodeOpts := raft.NodeOptions{ ID: securityConfig.ClientTLSCreds.NodeID(), Addr: oldNode.Address, @@ -334,6 +416,7 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo ClockSource: clockSource, SendTimeout: 10 * time.Second, TLSCredentials: securityConfig.ClientTLSCreds, + KeyRotator: kr, } ctx, cancel := context.WithCancel(context.Background()) @@ -345,16 +428,11 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo go func() { // After stopping, we should receive an error from Serve - assert.Error(t, s.Serve(wrappedListener)) + require.Error(t, s.Serve(wrappedListener)) }() healthServer.SetServingStatus("Raft", api.HealthCheckResponse_SERVING) - err := n.JoinAndStart(ctx) - require.NoError(t, err, "can't join cluster") - - go n.Run(ctx) - return &TestNode{ Node: n, Listener: wrappedListener, @@ -363,7 +441,20 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo StateDir: newNodeOpts.StateDir, cancel: cancel, Server: s, - } + KeyRotator: kr, + }, ctx +} + +// RestartNode restarts a raft test node +func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNode, forceNewCluster bool) *TestNode { + n, ctx := CopyNode(t, clockSource, oldNode, forceNewCluster, nil) + + err := n.Node.JoinAndStart(ctx) + require.NoError(t, err, "can't join cluster") + + go n.Node.Run(ctx) + + return n } // NewRaftCluster creates a new raft cluster with 3 nodes for testing diff --git a/node/node.go b/node/node.go index 53195d7a90..2cc3b4bb25 100644 --- a/node/node.go +++ b/node/node.go @@ -21,6 +21,7 @@ import ( "github.com/docker/swarmkit/ioutils" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/remotes" "github.com/docker/swarmkit/xnet" "github.com/pkg/errors" @@ -34,6 +35,10 @@ const stateFilename = "state.json" var ( errNodeStarted = errors.New("node: already started") errNodeNotStarted = errors.New("node: not started") + certDirectory = "certificates" + + // ErrInvalidUnlockKey is returned when we can't decrypt the TLS certificate + ErrInvalidUnlockKey = errors.New("node is locked, and needs a valid unlock key") ) // Config provides values for a Node. @@ -81,6 +86,14 @@ type Config struct { // HeartbeatTick defines the amount of ticks between each // heartbeat sent to other members for health-check purposes HeartbeatTick uint32 + + // AutoLockManagers determines whether or not an unlock key will be generated + // when bootstrapping a new cluster for the first time + AutoLockManagers bool + + // UnlockKey is the key to unlock a node - used for decrypting at rest. This + // only applies to nodes that have already joined a cluster. + UnlockKey []byte } // Node implements the primary node functionality for a member of a swarm @@ -106,6 +119,7 @@ type Node struct { agent *agent.Agent manager *manager.Manager notifyNodeChange chan *api.Node // used to send role updates from the dispatcher api on promotion/demotion + unlockKey []byte } // RemoteAPIAddr returns address on which remote manager api listens. @@ -150,12 +164,18 @@ func New(c *Config) (*Node, error) { ready: make(chan struct{}), certificateRequested: make(chan struct{}), notifyNodeChange: make(chan *api.Node, 1), + unlockKey: c.UnlockKey, } + + if n.config.JoinAddr != "" || n.config.ForceNewCluster { + n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename)) + if n.config.JoinAddr != "" { + n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight) + } + } + n.roleCond = sync.NewCond(n.RLocker()) n.connCond = sync.NewCond(n.RLocker()) - if err := n.loadCertificates(); err != nil { - return nil, err - } return n, nil } @@ -189,46 +209,7 @@ func (n *Node) run(ctx context.Context) (err error) { } }() - // NOTE: When this node is created by NewNode(), our nodeID is set if - // n.loadCertificates() succeeded in loading TLS credentials. - if n.config.JoinAddr == "" && n.nodeID == "" { - if err := n.bootstrapCA(); err != nil { - return err - } - } - - if n.config.JoinAddr != "" || n.config.ForceNewCluster { - n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename)) - if n.config.JoinAddr != "" { - n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight) - } - } - - // Obtain new certs and setup TLS certificates renewal for this node: - // - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued - // - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows - // us to display the ID before the certificate gets issued (for potential approval). - // - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate. - // - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay - // up to date. - issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1) - go func() { - select { - case <-ctx.Done(): - case resp := <-issueResponseChan: - log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{ - "node.id": resp.NodeID, - }).Debugf("requesting certificate") - n.Lock() - n.nodeID = resp.NodeID - n.nodeMembership = resp.NodeMembership - n.Unlock() - close(n.certificateRequested) - } - }() - - certDir := filepath.Join(n.config.StateDir, "certificates") - securityConfig, err := ca.LoadOrCreateSecurityConfig(ctx, certDir, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan) + securityConfig, err := n.loadSecurityConfig(ctx) if err != nil { return err } @@ -244,10 +225,6 @@ func (n *Node) run(ctx context.Context) (err error) { } defer db.Close() - if err := n.loadCertificates(); err != nil { - return err - } - forceCertRenewal := make(chan struct{}) renewCert := func() { select { @@ -289,7 +266,7 @@ func (n *Node) run(ctx context.Context) (err error) { } }() - updates := ca.RenewTLSConfig(ctx, securityConfig, certDir, n.remotes, forceCertRenewal) + updates := ca.RenewTLSConfig(ctx, securityConfig, n.remotes, forceCertRenewal) go func() { for { select { @@ -515,40 +492,100 @@ func (n *Node) Remotes() []api.Peer { return remotes } -func (n *Node) loadCertificates() error { - certDir := filepath.Join(n.config.StateDir, "certificates") - rootCA, err := ca.GetLocalRootCA(certDir) - if err != nil { - if err == ca.ErrNoLocalRootCA { - return nil +func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) { + paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory)) + var securityConfig *ca.SecurityConfig + + krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{}) + if err := krw.Migrate(); err != nil { + return nil, err + } + + // Check if we already have a valid certificates on disk. + rootCA, err := ca.GetLocalRootCA(paths.RootCA) + if err != nil && err != ca.ErrNoLocalRootCA { + return nil, err + } + if err == nil { + clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw) + _, ok := errors.Cause(err).(ca.ErrInvalidKEK) + switch { + case err == nil: + securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds) + log.G(ctx).Debug("loaded CA and TLS certificates") + case ok: + return nil, ErrInvalidUnlockKey + case os.IsNotExist(err): + break + default: + return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert) } - return err } - configPaths := ca.NewConfigPaths(certDir) - clientTLSCreds, _, err := ca.LoadTLSCreds(rootCA, configPaths.Node) - if err != nil { - if os.IsNotExist(err) { - return nil + + if securityConfig == nil { + if n.config.JoinAddr == "" { + // if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key + n.unlockKey = nil + if n.config.AutoLockManagers { + n.unlockKey = encryption.GenerateSecretKey() + } + krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{}) + rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + if err != nil { + return nil, err + } + log.G(ctx).Debug("generated CA key and certificate") + } else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk + rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes) + if err != nil { + return nil, err + } + log.G(ctx).Debug("downloaded CA certificate") } - return errors.Wrapf(err, "error while loading TLS Certificate in %s", configPaths.Node.Cert) + // Obtain new certs and setup TLS certificates renewal for this node: + // - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued + // - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows + // us to display the ID before the certificate gets issued (for potential approval). + // - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate. + // - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay + // up to date. + issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1) + go func() { + select { + case <-ctx.Done(): + case resp := <-issueResponseChan: + log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{ + "node.id": resp.NodeID, + }).Debugf("loaded TLS certificate") + n.Lock() + n.nodeID = resp.NodeID + n.nodeMembership = resp.NodeMembership + n.Unlock() + close(n.certificateRequested) + } + }() + + // LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS + // certificates and write them to disk + securityConfig, err = ca.LoadOrCreateSecurityConfig( + ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw) + if err != nil { + if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok { + return nil, ErrInvalidUnlockKey + } + return nil, err + } } - // todo: try csr if no cert or store nodeID/role in some other way + n.Lock() - n.role = clientTLSCreds.Role() - n.nodeID = clientTLSCreds.NodeID() + n.role = securityConfig.ClientTLSCreds.Role() + n.nodeID = securityConfig.ClientTLSCreds.NodeID() n.nodeMembership = api.NodeMembershipAccepted n.roleCond.Broadcast() n.Unlock() - return nil -} - -func (n *Node) bootstrapCA() error { - if err := ca.BootstrapCluster(filepath.Join(n.config.StateDir, "certificates")); err != nil { - return err - } - return n.loadCertificates() + return securityConfig, nil } func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error { @@ -626,13 +663,15 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig ListenAddr: n.config.ListenRemoteAPI, AdvertiseAddr: n.config.AdvertiseRemoteAPI, }, - ControlAPI: n.config.ListenControlAPI, - SecurityConfig: securityConfig, - ExternalCAs: n.config.ExternalCAs, - JoinRaft: remoteAddr.Addr, - StateDir: n.config.StateDir, - HeartbeatTick: n.config.HeartbeatTick, - ElectionTick: n.config.ElectionTick, + ControlAPI: n.config.ListenControlAPI, + SecurityConfig: securityConfig, + ExternalCAs: n.config.ExternalCAs, + JoinRaft: remoteAddr.Addr, + StateDir: n.config.StateDir, + HeartbeatTick: n.config.HeartbeatTick, + ElectionTick: n.config.ElectionTick, + AutoLockManagers: n.config.AutoLockManagers, + UnlockKey: n.unlockKey, }) if err != nil { return err diff --git a/node/node_test.go b/node/node_test.go new file mode 100644 index 0000000000..ef74912ef5 --- /dev/null +++ b/node/node_test.go @@ -0,0 +1,238 @@ +package node + +import ( + "context" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/ca" + cautils "github.com/docker/swarmkit/ca/testutils" + "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/manager/state/store" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +// If there is nothing on disk and no join addr, we create a new CA and a new set of TLS certs. +// If AutoLockManagers is enabled, the TLS key is encrypted with a randomly generated lock key. +func TestLoadSecurityConfigNewNode(t *testing.T) { + for _, autoLockManagers := range []bool{true, false} { + tempdir, err := ioutil.TempDir("", "test-new-node") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + + node, err := New(&Config{ + StateDir: tempdir, + AutoLockManagers: autoLockManagers, + }) + require.NoError(t, err) + securityConfig, err := node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + unencryptedReader := ca.NewKeyReadWriter(paths.Node, nil, nil) + _, _, err = unencryptedReader.Read() + if !autoLockManagers { + require.NoError(t, err) + } else { + require.IsType(t, ca.ErrInvalidKEK{}, err) + } + } +} + +// If there's only a root CA on disk (no TLS certs), and no join addr, we create a new CA +// and a new set of TLS certs. Similarly if there's only a TLS cert and key, and no CA. +func TestLoadSecurityConfigPartialCertsOnDisk(t *testing.T) { + tempdir, err := ioutil.TempDir("", "test-new-node") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + require.NoError(t, err) + + node, err := New(&Config{ + StateDir: tempdir, + }) + require.NoError(t, err) + securityConfig, err := node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + cert, key, err := securityConfig.KeyReader().Read() + require.NoError(t, err) + + // a new CA was generated because no existing TLS certs were present + require.NotEqual(t, rootCA.Cert, securityConfig.RootCA().Cert) + + // if the TLS key and cert are on disk, but there's no CA, a new CA and TLS + // key+cert are generated + require.NoError(t, os.RemoveAll(paths.RootCA.Cert)) + + node, err = New(&Config{ + StateDir: tempdir, + }) + require.NoError(t, err) + securityConfig, err = node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + newCert, newKey, err := securityConfig.KeyReader().Read() + require.NoError(t, err) + require.NotEqual(t, cert, newCert) + require.NotEqual(t, key, newKey) +} + +// If there are CAs and TLS certs on disk, it tries to load and fails if there +// are any errors, even if a join token is provided. +func TestLoadSecurityConfigLoadFromDisk(t *testing.T) { + tempdir, err := ioutil.TempDir("", "test-load-node-tls") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + + tc := cautils.NewTestCA(t) + defer tc.Stop() + peer, err := tc.Remotes.Select() + require.NoError(t, err) + + // Load successfully with valid passphrase + rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + require.NoError(t, err) + krw := ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil) + require.NoError(t, err) + _, err = rootCA.IssueAndSaveNewCertificates(krw, identity.NewID(), ca.WorkerRole, identity.NewID()) + require.NoError(t, err) + + node, err := New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + UnlockKey: []byte("passphrase"), + }) + require.NoError(t, err) + securityConfig, err := node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + // Invalid passphrase + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.Equal(t, ErrInvalidUnlockKey, err) + + // Invalid CA + rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + require.NoError(t, err) + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + UnlockKey: []byte("passphrase"), + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.IsType(t, x509.UnknownAuthorityError{}, errors.Cause(err)) +} + +// If there is no CA, and a join addr is provided, one is downloaded from the +// join server. If there is a CA, it is just loaded from disk. The TLS key and +// cert are also downloaded. +func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) { + tempdir, err := ioutil.TempDir("", "test-join-node") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + + // join addr is invalid + node, err := New(&Config{ + StateDir: tempdir, + JoinAddr: "127.0.0.1:12", + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.Error(t, err) + + tc := cautils.NewTestCA(t) + defer tc.Stop() + + peer, err := tc.Remotes.Select() + require.NoError(t, err) + + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + + // the TLS key and cert were written to disk unencrypted + _, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read() + require.NoError(t, err) + + // remove the TLS cert and key, and mark the root CA cert so that we will + // know if it gets replaced + require.NoError(t, os.Remove(paths.Node.Cert)) + require.NoError(t, os.Remove(paths.Node.Key)) + certBytes, err := ioutil.ReadFile(paths.RootCA.Cert) + require.NoError(t, err) + pemBlock, _ := pem.Decode(certBytes) + require.NotNil(t, pemBlock) + pemBlock.Headers["marked"] = "true" + certBytes = pem.EncodeToMemory(pemBlock) + require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, certBytes, 0644)) + + // also make sure the new set gets downloaded and written to disk with a passphrase + // by updating the memory store with manager autolock on and an unlock key + require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error { + clusters, err := store.FindClusters(tx, store.All) + require.NoError(t, err) + require.Len(t, clusters, 1) + + newCluster := clusters[0].Copy() + newCluster.Spec.EncryptionConfig.AutoLockManagers = true + newCluster.UnlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: []byte("passphrase"), + }} + return store.UpdateCluster(tx, newCluster) + })) + + // Join with without any passphrase - this should be fine, because the TLS + // key is downloaded and then loaded just fine. However, it *is* written + // to disk encrypted. + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + + // make sure the CA cert has not been replaced + readCertBytes, err := ioutil.ReadFile(paths.RootCA.Cert) + require.NoError(t, err) + require.Equal(t, certBytes, readCertBytes) + + // the TLS node cert and key were saved to disk encrypted, though + _, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read() + require.Error(t, err) + _, _, err = ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil).Read() + require.NoError(t, err) +}