From 491732025f16eaf56f846e5316d2b16457c400e8 Mon Sep 17 00:00:00 2001 From: cyli Date: Fri, 21 Oct 2016 22:28:58 -0700 Subject: [PATCH 1/3] Refactor the ca package: 1. There is a single object that knows how to read/write TLS keys and certs. This object knows how to encrypt/decrypt the TLS key, and preserve any headers stored on the key as specified by a PEMKeyHeaders interface. 2. No longer write the Root CA key to disk - it is stored inside raft only. Refactor the node TLS loading/bootstrapping as well to live in a single function. Signed-off-by: cyli --- ca/certificates.go | 153 ++---------- ca/certificates_test.go | 174 +++++--------- ca/config.go | 270 ++++++++++----------- ca/config_test.go | 225 ++++++++--------- ca/keyreadwriter.go | 388 ++++++++++++++++++++++++++++++ ca/keyreadwriter_test.go | 414 ++++++++++++++++++++++++++++++++ ca/server_test.go | 21 +- ca/testutils/cautils.go | 34 ++- ca/testutils/externalutils.go | 2 +- ca/transport_test.go | 8 +- cmd/external-ca-example/main.go | 6 +- node/node.go | 182 ++++++++------ node/node_test.go | 207 ++++++++++++++++ 13 files changed, 1473 insertions(+), 611 deletions(-) create mode 100644 ca/keyreadwriter.go create mode 100644 ca/keyreadwriter_test.go create mode 100644 node/node_test.go diff --git a/ca/certificates.go b/ca/certificates.go index 19022d58ef..b4aa5fcf83 100644 --- a/ca/certificates.go +++ b/ca/certificates.go @@ -23,7 +23,6 @@ import ( "github.com/docker/distribution/digest" "github.com/docker/go-events" "github.com/docker/swarmkit/api" - "github.com/docker/swarmkit/identity" "github.com/docker/swarmkit/ioutils" "github.com/docker/swarmkit/remotes" "github.com/pkg/errors" @@ -122,8 +121,8 @@ func (rca *RootCA) CanSign() bool { // IssueAndSaveNewCertificates generates a new key-pair, signs it with the local root-ca, and returns a // tls certificate -func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org string) (*tls.Certificate, error) { - csr, key, err := GenerateAndWriteNewKey(paths) +func (rca *RootCA) IssueAndSaveNewCertificates(kw KeyWriter, cn, ou, org string) (*tls.Certificate, error) { + csr, key, err := GenerateNewCSR() if err != nil { return nil, errors.Wrap(err, "error when generating new node certs") } @@ -138,20 +137,13 @@ func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org stri return nil, errors.Wrap(err, "failed to sign node certificate") } - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) + // Create a valid TLSKeyPair out of the PEM encoded private key and certificate + tlsKeyPair, err := tls.X509KeyPair(certChain, key) if err != nil { return nil, err } - // Write the chain to disk - if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil { - return nil, err - } - - // Create a valid TLSKeyPair out of the PEM encoded private key and certificate - tlsKeyPair, err := tls.X509KeyPair(certChain, key) - if err != nil { + if err := kw.Write(certChain, key, nil); err != nil { return nil, err } @@ -160,11 +152,9 @@ func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org stri // RequestAndSaveNewCertificates gets new certificates issued, either by signing them locally if a signer is // available, or by requesting them from the remote server at remoteAddr. -func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths CertPaths, token string, remotes remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) { - // Create a new key/pair and CSR for the new manager - // Write the new CSR and the new key to a temporary location so we can survive crashes on rotation - tempPaths := genTempPaths(paths) - csr, key, err := GenerateAndWriteNewKey(tempPaths) +func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, kw KeyWriter, token string, r remotes.Remotes, transport credentials.TransportCredentials, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) { + // Create a new key/pair and CSR + csr, key, err := GenerateNewCSR() if err != nil { return nil, errors.Wrap(err, "error when generating new node certs") } @@ -174,7 +164,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert // responding properly (for example, it may have just been demoted). var signedCert []byte for i := 0; i != 5; i++ { - signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, remotes, transport, nodeInfo) + signedCert, err = GetRemoteSignedCertificate(ctx, csr, token, rca.Pool, r, transport, nodeInfo) if err == nil { break } @@ -184,7 +174,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert } // Доверяй, но проверяй. - // Before we overwrite our local certificate, let's make sure the server gave us one that is valid + // Before we overwrite our local key + certificate, let's make sure the server gave us one that is valid // Create an X509Cert so we can .Verify() certBlock, _ := pem.Decode(signedCert) if certBlock == nil { @@ -209,19 +199,7 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths Cert return nil, err } - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) - if err != nil { - return nil, err - } - - // Write the chain to disk - if err := ioutils.AtomicWriteFile(paths.Cert, signedCert, 0644); err != nil { - return nil, err - } - - // Move the new key to the final location - if err := os.Rename(tempPaths.Key, paths.Key); err != nil { + if err := kw.Write(signedCert, key, nil); err != nil { return nil, err } @@ -388,11 +366,9 @@ func ensureCertKeyMatch(cert *x509.Certificate, key crypto.PublicKey) error { // GetLocalRootCA validates if the contents of the file are a valid self-signed // CA certificate, and returns the PEM-encoded Certificate if so -func GetLocalRootCA(baseDir string) (RootCA, error) { - paths := NewConfigPaths(baseDir) - +func GetLocalRootCA(paths CertPaths) (RootCA, error) { // Check if we have a Certificate file - cert, err := ioutil.ReadFile(paths.RootCA.Cert) + cert, err := ioutil.ReadFile(paths.Cert) if err != nil { if os.IsNotExist(err) { err = ErrNoLocalRootCA @@ -401,7 +377,7 @@ func GetLocalRootCA(baseDir string) (RootCA, error) { return RootCA{}, err } - key, err := ioutil.ReadFile(paths.RootCA.Key) + key, err := ioutil.ReadFile(paths.Key) if err != nil { if !os.IsNotExist(err) { return RootCA{}, err @@ -481,9 +457,9 @@ func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootC return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil } -// CreateAndWriteRootCA creates a Certificate authority for a new Swarm Cluster, potentially +// CreateRootCA creates a Certificate authority for a new Swarm Cluster, potentially // overwriting any existing CAs. -func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) { +func CreateRootCA(rootCN string, paths CertPaths) (RootCA, error) { // Create a simple CSR for the CA using the default CA validator and policy req := cfcsr.CertificateRequest{ CN: rootCN, @@ -497,99 +473,17 @@ func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) { return RootCA{}, err } - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) + rootCA, err := NewRootCA(cert, key, DefaultNodeCertExpiration) if err != nil { return RootCA{}, err } - // Write the Private Key and Certificate to disk, using decent permissions - if err := ioutils.AtomicWriteFile(paths.Cert, cert, 0644); err != nil { - return RootCA{}, err - } - if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil { + // save the cert to disk + if err := saveRootCA(rootCA, paths); err != nil { return RootCA{}, err } - return NewRootCA(cert, key, DefaultNodeCertExpiration) -} - -// BootstrapCluster receives a directory and creates both new Root CA key material -// and a ManagerRole key/certificate pair to be used by the initial cluster manager -func BootstrapCluster(baseCertDir string) error { - paths := NewConfigPaths(baseCertDir) - - rootCA, err := CreateAndWriteRootCA(rootCN, paths.RootCA) - if err != nil { - return err - } - - nodeID := identity.NewID() - newOrg := identity.NewID() - _, err = GenerateAndSignNewTLSCert(rootCA, nodeID, ManagerRole, newOrg, paths.Node) - - return err -} - -// GenerateAndSignNewTLSCert creates a new keypair, signs the certificate using signer, -// and saves the certificate and key to disk. This method is used to bootstrap the first -// manager TLS certificates. -func GenerateAndSignNewTLSCert(rootCA RootCA, cn, ou, org string, paths CertPaths) (*tls.Certificate, error) { - // Generate and new keypair and CSR - csr, key, err := generateNewCSR() - if err != nil { - return nil, err - } - - // Obtain a signed Certificate - certChain, err := rootCA.ParseValidateAndSignCSR(csr, cn, ou, org) - if err != nil { - return nil, errors.Wrap(err, "failed to sign node certificate") - } - - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Cert), 0755) - if err != nil { - return nil, err - } - - // Write both the chain and key to disk - if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil { - return nil, err - } - if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil { - return nil, err - } - - // Load a valid tls.Certificate from the chain and the key - serverCert, err := tls.X509KeyPair(certChain, key) - if err != nil { - return nil, err - } - - return &serverCert, nil -} - -// GenerateAndWriteNewKey generates a new pub/priv key pair, writes it to disk -// and returns the CSR and the private key material -func GenerateAndWriteNewKey(paths CertPaths) (csr, key []byte, err error) { - // Generate a new key pair - csr, key, err = generateNewCSR() - if err != nil { - return - } - - // Ensure directory exists - err = os.MkdirAll(filepath.Dir(paths.Key), 0755) - if err != nil { - return - } - - if err = ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil { - return - } - - return + return rootCA, nil } // GetRemoteSignedCertificate submits a CSR to a remote CA server address, @@ -681,10 +575,10 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r } // readCertValidity returns the certificate issue and expiration time -func readCertValidity(paths CertPaths) (time.Time, time.Time, error) { +func readCertValidity(kr KeyReader) (time.Time, time.Time, error) { var zeroTime time.Time // Read the Cert - cert, err := ioutil.ReadFile(paths.Cert) + cert, _, err := kr.Read() if err != nil { return zeroTime, zeroTime, err } @@ -714,7 +608,8 @@ func saveRootCA(rootCA RootCA, paths CertPaths) error { return ioutils.AtomicWriteFile(paths.Cert, rootCA.Cert, 0644) } -func generateNewCSR() (csr, key []byte, err error) { +// GenerateNewCSR returns a newly generated key and CSR signed with said key +func GenerateNewCSR() (csr, key []byte, err error) { req := &cfcsr.CertificateRequest{ KeyRequest: cfcsr.NewBasicKeyRequest(), } diff --git a/ca/certificates_test.go b/ca/certificates_test.go index 861be61360..9ce6fe3cd0 100644 --- a/ca/certificates_test.go +++ b/ca/certificates_test.go @@ -23,6 +23,7 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/phayes/permbits" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/net/context" ) @@ -42,34 +43,33 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestCreateAndWriteRootCA(t *testing.T) { +func TestCreateRootCA(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) - _, err = ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + _, err = ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) perms, err := permbits.Stat(paths.RootCA.Cert) assert.NoError(t, err) assert.False(t, perms.GroupWrite()) assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.RootCA.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) + + _, err = permbits.Stat(paths.RootCA.Key) + assert.True(t, os.IsNotExist(err)) } -func TestCreateAndWriteRootCAExpiry(t *testing.T) { +func TestCreateRootCAExpiry(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) // Convert the certificate into an object to create a RootCA @@ -89,26 +89,26 @@ func TestGetLocalRootCA(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) // First, try to load the local Root CA with the certificate missing. - _, err = ca.GetLocalRootCA(tempBaseDir) + _, err = ca.GetLocalRootCA(paths.RootCA) assert.Equal(t, ca.ErrNoLocalRootCA, err) // Create the local Root CA to ensure that we can reload it correctly. - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.True(t, rootCA.CanSign()) assert.NoError(t, err) - rootCA2, err := ca.GetLocalRootCA(tempBaseDir) + // No private key here + rootCA2, err := ca.GetLocalRootCA(paths.RootCA) assert.NoError(t, err) - assert.True(t, rootCA2.CanSign()) - assert.Equal(t, rootCA, rootCA2) + assert.Equal(t, rootCA.Cert, rootCA2.Cert) + assert.False(t, rootCA2.CanSign()) - // Try again, this time without a private key. - assert.NoError(t, os.Remove(paths.RootCA.Key)) - - rootCA3, err := ca.GetLocalRootCA(tempBaseDir) + // write private key and assert we can load it and sign + assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, rootCA.Key, os.FileMode(0600))) + rootCA3, err := ca.GetLocalRootCA(paths.RootCA) assert.NoError(t, err) - assert.False(t, rootCA3.CanSign()) assert.Equal(t, rootCA.Cert, rootCA3.Cert) + assert.True(t, rootCA3.CanSign()) // Try with a private key that does not match the CA cert public key. privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) @@ -121,62 +121,43 @@ func TestGetLocalRootCA(t *testing.T) { }) assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, privKeyPem, os.FileMode(0600))) - _, err = ca.GetLocalRootCA(tempBaseDir) + _, err = ca.GetLocalRootCA(paths.RootCA) assert.EqualError(t, err, "certificate key mismatch") } -func TestGenerateAndSignNewTLSCert(t *testing.T) { +func TestGetLocalRootCAInvalidCert(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) - assert.NoError(t, err) + // Write some garbage to the CA cert + require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, []byte(`-----BEGIN CERTIFICATE-----\n +some random garbage\n +-----END CERTIFICATE-----`), 0644)) - _, err = ca.GenerateAndSignNewTLSCert(rootCA, "CN", "OU", "ORG", paths.Node) - assert.NoError(t, err) - - perms, err := permbits.Stat(paths.Node.Cert) - assert.NoError(t, err) - assert.False(t, perms.GroupWrite()) - assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.Node.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) - - certBytes, err := ioutil.ReadFile(paths.Node.Cert) - assert.NoError(t, err) - certs, err := helpers.ParseCertificatesPEM(certBytes) - assert.NoError(t, err) - assert.Len(t, certs, 2) - assert.Equal(t, "CN", certs[0].Subject.CommonName) - assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0]) - assert.Equal(t, "ORG", certs[0].Subject.Organization[0]) - assert.Equal(t, "rootCN", certs[1].Subject.CommonName) + _, err = ca.GetLocalRootCA(paths.RootCA) + require.Error(t, err) } -func TestGenerateAndWriteNewKey(t *testing.T) { +func TestGetLocalRootCAInvalidKey(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) paths := ca.NewConfigPaths(tempBaseDir) + // Create the local Root CA to ensure that we can reload it correctly. + _, err = ca.CreateRootCA("rootCN", paths.RootCA) + require.NoError(t, err) - csr, key, err := ca.GenerateAndWriteNewKey(paths.Node) - assert.NoError(t, err) - assert.NotNil(t, csr) - assert.NotNil(t, key) - - perms, err := permbits.Stat(paths.Node.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) + // Write some garbage to the root key - this will cause the loading to fail + require.NoError(t, ioutil.WriteFile(paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n +some random garbage\n +-----END EC PRIVATE KEY-----`), 0600)) - _, err = helpers.ParseCSRPEM(csr) - assert.NoError(t, err) + _, err = ca.GetLocalRootCA(paths.RootCA) + require.Error(t, err) } func TestEncryptECPrivateKey(t *testing.T) { @@ -184,9 +165,7 @@ func TestEncryptECPrivateKey(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(tempBaseDir) - paths := ca.NewConfigPaths(tempBaseDir) - - _, key, err := ca.GenerateAndWriteNewKey(paths.Node) + _, key, err := ca.GenerateNewCSR() assert.NoError(t, err) encryptedKey, err := ca.EncryptECPrivateKey(key, "passphrase") assert.NoError(t, err) @@ -204,10 +183,10 @@ func TestParseValidateAndSignCSR(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) - csr, _, err := ca.GenerateAndWriteNewKey(paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) signedCert, err := rootCA.ParseValidateAndSignCSR(csr, "CN", "OU", "ORG") @@ -232,7 +211,7 @@ func TestParseValidateAndSignMaliciousCSR(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) req := &cfcsr.CertificateRequest{ @@ -308,7 +287,7 @@ func TestRequestAndSaveNewCertificates(t *testing.T) { info := make(chan api.IssueNodeCertificateResponse, 1) // Copy the current RootCA without the signer rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool} - cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.Paths.Node, tc.WorkerToken, tc.Remotes, nil, info) + cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.ManagerToken, tc.Remotes, nil, info) assert.NoError(t, err) assert.NotNil(t, cert) perms, err := permbits.Stat(tc.Paths.Node.Cert) @@ -316,6 +295,11 @@ func TestRequestAndSaveNewCertificates(t *testing.T) { assert.False(t, perms.GroupWrite()) assert.False(t, perms.OtherWrite()) assert.NotEmpty(t, <-info) + + // the key should be unencrypted + unencryptedKeyReader := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + _, _, err = unencryptedKeyReader.Read() + require.NoError(t, err) } func TestIssueAndSaveNewCertificates(t *testing.T) { @@ -323,7 +307,7 @@ func TestIssueAndSaveNewCertificates(t *testing.T) { defer tc.Stop() // Test the creation of a manager certificate - cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.ManagerRole, tc.Organization) + cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.ManagerRole, tc.Organization) assert.NoError(t, err) assert.NotNil(t, cert) perms, err := permbits.Stat(tc.Paths.Node.Cert) @@ -345,7 +329,7 @@ func TestIssueAndSaveNewCertificates(t *testing.T) { assert.Contains(t, certs[0].DNSNames, "swarm-manager") // Test the creation of a worker node cert - cert, err = tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.WorkerRole, tc.Organization) + cert, err = tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.WorkerRole, tc.Organization) assert.NoError(t, err) assert.NotNil(t, cert) perms, err = permbits.Stat(tc.Paths.Node.Cert) @@ -372,7 +356,7 @@ func TestGetRemoteSignedCertificate(t *testing.T) { defer tc.Stop() // Create a new CSR to be signed - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) certs, err := ca.GetRemoteSignedCertificate(context.Background(), csr, tc.ManagerToken, tc.RootCA.Pool, tc.Remotes, nil, nil) @@ -404,7 +388,7 @@ func TestGetRemoteSignedCertificateNodeInfo(t *testing.T) { defer tc.Stop() // Create a new CSR to be signed - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) info := make(chan api.IssueNodeCertificateResponse, 1) @@ -421,7 +405,7 @@ func TestGetRemoteSignedCertificateWithPending(t *testing.T) { defer tc.Stop() // Create a new CSR to be signed - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) updates, cancel := state.Watch(tc.MemoryStore.WatchQueue(), state.EventCreateNode{}) @@ -448,35 +432,6 @@ func TestGetRemoteSignedCertificateWithPending(t *testing.T) { assert.NoError(t, <-completed) } -func TestBootstrapCluster(t *testing.T) { - tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") - assert.NoError(t, err) - defer os.RemoveAll(tempBaseDir) - - paths := ca.NewConfigPaths(tempBaseDir) - - err = ca.BootstrapCluster(tempBaseDir) - assert.NoError(t, err) - - perms, err := permbits.Stat(paths.RootCA.Cert) - assert.NoError(t, err) - assert.False(t, perms.GroupWrite()) - assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.RootCA.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) - - perms, err = permbits.Stat(paths.Node.Cert) - assert.NoError(t, err) - assert.False(t, perms.GroupWrite()) - assert.False(t, perms.OtherWrite()) - perms, err = permbits.Stat(paths.Node.Key) - assert.NoError(t, err) - assert.False(t, perms.GroupRead()) - assert.False(t, perms.OtherRead()) -} - func TestNewRootCA(t *testing.T) { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) @@ -484,7 +439,7 @@ func TestNewRootCA(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, ca.DefaultNodeCertExpiration) @@ -499,12 +454,12 @@ func TestNewRootCABundle(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - // Write one root CA to disk, keep the bytes - secondRootCA, err := ca.CreateAndWriteRootCA("rootCN2", paths.RootCA) + // make one rootCA + secondRootCA, err := ca.CreateRootCA("rootCN2", paths.RootCA) assert.NoError(t, err) - // Overwrite the first root CA on disk - firstRootCA, err := ca.CreateAndWriteRootCA("rootCN1", paths.RootCA) + // make a second root CA + firstRootCA, err := ca.CreateRootCA("rootCN1", paths.RootCA) assert.NoError(t, err) // Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle @@ -517,14 +472,9 @@ func TestNewRootCABundle(t *testing.T) { assert.Equal(t, bundle, newRootCA.Cert) assert.Equal(t, 2, len(newRootCA.Pool.Subjects())) - // Now load the bundle from disk - diskRootCA, err := ca.GetLocalRootCA(tempBaseDir) - assert.NoError(t, err) - assert.Equal(t, bundle, diskRootCA.Cert) - assert.Equal(t, 2, len(diskRootCA.Pool.Subjects())) - - // If I use GenerateAndSignNewTLSCert to sign certs, I'll get the correct CA in the chain - _, err = ca.GenerateAndSignNewTLSCert(diskRootCA, "CN", "OU", "ORG", paths.Node) + // If I use newRootCA's IssueAndSaveNewCertificates to sign certs, I'll get the correct CA in the chain + kw := ca.NewKeyReadWriter(paths.Node, nil, nil) + _, err = newRootCA.IssueAndSaveNewCertificates(kw, "CN", "OU", "ORG") assert.NoError(t, err) certBytes, err := ioutil.ReadFile(paths.Node.Cert) @@ -546,14 +496,14 @@ func TestNewRootCANonDefaultExpiry(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, 1*time.Hour) assert.NoError(t, err) // Create and sign a new CSR - csr, _, err := ca.GenerateAndWriteNewKey(paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) cert, err := newRootCA.ParseValidateAndSignCSR(csr, "CN", ca.ManagerRole, "ORG") assert.NoError(t, err) @@ -588,7 +538,7 @@ func TestNewRootCAWithPassphrase(t *testing.T) { paths := ca.NewConfigPaths(tempBaseDir) - rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA) + rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA) assert.NoError(t, err) // Ensure that we're encrypting the Key bytes out of NewRoot if there diff --git a/ca/config.go b/ca/config.go index 872ab5e1cb..f6b5d150f0 100644 --- a/ca/config.go +++ b/ca/config.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io/ioutil" "math/big" "math/rand" "path/filepath" @@ -33,7 +32,8 @@ const ( nodeTLSKeyFilename = "swarm-node.key" nodeCSRFilename = "swarm-node.csr" - rootCN = "swarm-ca" + // DefaultRootCN represents the root CN that we should create roots CAs with by default + DefaultRootCN = "swarm-ca" // ManagerRole represents the Manager node type, and is used for authorization to endpoints ManagerRole = "swarm-manager" // WorkerRole represents the Worker node type, and is used for authorization to endpoints @@ -54,8 +54,9 @@ const ( type SecurityConfig struct { mu sync.Mutex - rootCA *RootCA - externalCA *ExternalCA + rootCA *RootCA + externalCA *ExternalCA + keyReadWriter *KeyReadWriter ServerTLSCreds *MutableTLSCreds ClientTLSCreds *MutableTLSCreds @@ -69,7 +70,7 @@ type CertificateUpdate struct { } // NewSecurityConfig initializes and returns a new SecurityConfig. -func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig { +func NewSecurityConfig(rootCA *RootCA, krw *KeyReadWriter, clientTLSCreds, serverTLSCreds *MutableTLSCreds) *SecurityConfig { // Make a new TLS config for the external CA client without a // ServerName value set. clientTLSConfig := clientTLSCreds.Config() @@ -82,6 +83,7 @@ func NewSecurityConfig(rootCA *RootCA, clientTLSCreds, serverTLSCreds *MutableTL return &SecurityConfig{ rootCA: rootCA, + keyReadWriter: krw, externalCA: NewExternalCA(rootCA, externalCATLSConfig), ClientTLSCreds: clientTLSCreds, ServerTLSCreds: serverTLSCreds, @@ -96,6 +98,16 @@ func (s *SecurityConfig) RootCA() *RootCA { return s.rootCA } +// KeyWriter returns the object that can write keys to disk +func (s *SecurityConfig) KeyWriter() KeyWriter { + return s.keyReadWriter +} + +// KeyReader returns the object that can read keys from disk +func (s *SecurityConfig) KeyReader() KeyReader { + return s.keyReadWriter +} + // UpdateRootCA replaces the root CA with a new root CA based on the specified // certificate, key, and the number of hours the certificates issue should last. func (s *SecurityConfig) UpdateRootCA(cert, key []byte, certExpiry time.Duration) error { @@ -181,70 +193,63 @@ func getCAHashFromToken(token string) (digest.Digest, error) { return digest.ParseDigest(fmt.Sprintf("sha256:%0[1]*s", 64, digestInt.Text(16))) } -// LoadOrCreateSecurityConfig encapsulates the security logic behind joining a cluster. -// Every node requires at least a set of TLS certificates with which to join the cluster with. -// In the case of a manager, these certificates will be used both for client and server credentials. -func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse) (*SecurityConfig, error) { - ctx = log.WithModule(ctx, "tls") - paths := NewConfigPaths(baseCertDir) - +// DownloadRootCA tries to retrieve a remote root CA and matches the digest against the provided token. +func DownloadRootCA(ctx context.Context, paths CertPaths, token string, r remotes.Remotes) (RootCA, error) { + var rootCA RootCA + // Get a digest for the optional CA hash string that we've been provided + // If we were provided a non-empty string, and it is an invalid hash, return + // otherwise, allow the invalid digest through. var ( - rootCA RootCA - serverTLSCreds, clientTLSCreds *MutableTLSCreds - err error + d digest.Digest + err error ) - - // Check if we already have a CA certificate on disk. We need a CA to have a valid SecurityConfig - rootCA, err = GetLocalRootCA(baseCertDir) - switch err { - case nil: - log.G(ctx).Debug("loaded CA certificate") - case ErrNoLocalRootCA: - log.G(ctx).WithError(err).Debugf("failed to load local CA certificate") - - // Get a digest for the optional CA hash string that we've been provided - // If we were provided a non-empty string, and it is an invalid hash, return - // otherwise, allow the invalid digest through. - var d digest.Digest - if token != "" { - d, err = getCAHashFromToken(token) - if err != nil { - return nil, err - } - } - - // Get the remote CA certificate, verify integrity with the - // hash provided. Retry up to 5 times, in case the manager we - // first try to contact is not responding properly (it may have - // just been demoted, for example). - - for i := 0; i != 5; i++ { - rootCA, err = GetRemoteCA(ctx, d, remotes) - if err == nil { - break - } - log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate") - } + if token != "" { + d, err = getCAHashFromToken(token) if err != nil { - return nil, err + return RootCA{}, err } - - // Save root CA certificate to disk - if err = saveRootCA(rootCA, paths.RootCA); err != nil { - return nil, err + } + // Get the remote CA certificate, verify integrity with the + // hash provided. Retry up to 5 times, in case the manager we + // first try to contact is not responding properly (it may have + // just been demoted, for example). + + for i := 0; i != 5; i++ { + rootCA, err = GetRemoteCA(ctx, d, r) + if err == nil { + break } + log.G(ctx).WithError(err).Errorf("failed to retrieve remote root CA certificate") + } + if err != nil { + return RootCA{}, err + } - log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.RootCA.Cert) - default: - return nil, err + // Save root CA certificate to disk + if err = saveRootCA(rootCA, paths); err != nil { + return RootCA{}, err } + log.G(ctx).Debugf("retrieved remote CA certificate: %s", paths.Cert) + return rootCA, nil +} + +// LoadOrCreateSecurityConfig encapsulates the security logic behind joining a cluster. +// Every node requires at least a set of TLS certificates with which to join the cluster with. +// In the case of a manager, these certificates will be used both for client and server credentials. +func LoadOrCreateSecurityConfig(ctx context.Context, rootCA RootCA, token, proposedRole string, remotes remotes.Remotes, nodeInfo chan<- api.IssueNodeCertificateResponse, krw *KeyReadWriter) (*SecurityConfig, error) { + ctx = log.WithModule(ctx, "tls") + // At this point we've successfully loaded the CA details from disk, or // successfully downloaded them remotely. The next step is to try to // load our certificates. - clientTLSCreds, serverTLSCreds, err = LoadTLSCreds(rootCA, paths.Node) + clientTLSCreds, serverTLSCreds, err := LoadTLSCreds(rootCA, krw) if err != nil { - log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", paths.Node.Cert) + if _, ok := errors.Cause(err).(ErrInvalidKEK); ok { + return nil, err + } + + log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target()) var ( tlsKeyPair *tls.Certificate @@ -262,7 +267,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose NodeMembership: api.NodeMembershipAccepted, } } - tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(paths.Node, cn, proposedRole, org) + tlsKeyPair, err = rootCA.IssueAndSaveNewCertificates(krw, cn, proposedRole, org) if err != nil { log.G(ctx).WithFields(logrus.Fields{ "node.id": cn, @@ -278,7 +283,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose } else { // There was an error loading our Credentials, let's get a new certificate issued // Last argument is nil because at this point we don't have any valid TLS creds - tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, paths.Node, token, remotes, nil, nodeInfo) + tlsKeyPair, err = rootCA.RequestAndSaveNewCertificates(ctx, krw, token, remotes, nil, nodeInfo) if err != nil { log.G(ctx).WithError(err).Error("failed to request save new certificate") return nil, err @@ -299,7 +304,7 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose log.G(ctx).WithFields(logrus.Fields{ "node.id": clientTLSCreds.NodeID(), "node.role": clientTLSCreds.Role(), - }).Debugf("new node credentials generated: %s", paths.Node.Cert) + }).Debugf("new node credentials generated: %s", krw.Target()) } else { if nodeInfo != nil { nodeInfo <- api.IssueNodeCertificateResponse{ @@ -313,13 +318,66 @@ func LoadOrCreateSecurityConfig(ctx context.Context, baseCertDir, token, propose }).Debug("loaded node credentials") } - return NewSecurityConfig(&rootCA, clientTLSCreds, serverTLSCreds), nil + return NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds), nil +} + +// RenewTLSConfigNow gets a new TLS cert and key, and updates the security config if provided. This is similar to +// RenewTLSConfig, except while that monitors for expiry, and periodically renews, this renews once and is blocking +func RenewTLSConfigNow(ctx context.Context, s *SecurityConfig, r remotes.Remotes) error { + ctx = log.WithModule(ctx, "tls") + log := log.G(ctx).WithFields(logrus.Fields{ + "node.id": s.ClientTLSCreds.NodeID(), + "node.role": s.ClientTLSCreds.Role(), + }) + + // Let's request new certs. Renewals don't require a token. + rootCA := s.RootCA() + tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx, + s.KeyWriter(), + "", + r, + s.ClientTLSCreds, + nil) + if err != nil { + log.WithError(err).Errorf("failed to renew the certificate") + return err + } + + clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole) + if err != nil { + log.WithError(err).Errorf("failed to create a new client config") + return err + } + serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool) + if err != nil { + log.WithError(err).Errorf("failed to create a new server config") + return err + } + + if err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig); err != nil { + log.WithError(err).Errorf("failed to update the client credentials") + return err + } + + // Update the external CA to use the new client TLS + // config using a copy without a serverName specified. + s.externalCA.UpdateTLSConfig(&tls.Config{ + Certificates: clientTLSConfig.Certificates, + RootCAs: clientTLSConfig.RootCAs, + MinVersion: tls.VersionTLS12, + }) + + if err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig); err != nil { + log.WithError(err).Errorf("failed to update the server TLS credentials") + return err + } + + return nil } // RenewTLSConfig will continuously monitor for the necessity of renewing the local certificates, either by // issuing them locally if key-material is available, or requesting them from a remote CA. -func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate { - paths := NewConfigPaths(baseCertDir) +func RenewTLSConfig(ctx context.Context, s *SecurityConfig, remotes remotes.Remotes, renew <-chan struct{}) <-chan CertificateUpdate { updates := make(chan CertificateUpdate) go func() { @@ -337,10 +395,10 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, // Since the expiration of the certificate is managed remotely we should update our // retry timer on every iteration of this loop. // Retrieve the current certificate expiration information. - validFrom, validUntil, err := readCertValidity(paths.Node) + validFrom, validUntil, err := readCertValidity(s.KeyReader()) if err != nil { // We failed to read the expiration, let's stick with the starting default - log.Errorf("failed to read the expiration of the TLS certificate in: %s", paths.Node.Cert) + log.Errorf("failed to read the expiration of the TLS certificate in: %s", s.KeyReader().Target()) updates <- CertificateUpdate{Err: errors.New("failed to read certificate expiration")} } else { // If we have an expired certificate, we let's stick with the starting default in @@ -368,52 +426,12 @@ func RenewTLSConfig(ctx context.Context, s *SecurityConfig, baseCertDir string, return } - // Let's request new certs. Renewals don't require a token. - rootCA := s.RootCA() - tlsKeyPair, err := rootCA.RequestAndSaveNewCertificates(ctx, - paths.Node, - "", - remotes, - s.ClientTLSCreds, - nil) - if err != nil { - log.WithError(err).Errorf("failed to renew the certificate") - updates <- CertificateUpdate{Err: err} - continue - } - - clientTLSConfig, err := NewClientTLSConfig(tlsKeyPair, rootCA.Pool, CARole) - if err != nil { - log.WithError(err).Errorf("failed to create a new client config") - updates <- CertificateUpdate{Err: err} - } - serverTLSConfig, err := NewServerTLSConfig(tlsKeyPair, rootCA.Pool) - if err != nil { - log.WithError(err).Errorf("failed to create a new server config") - updates <- CertificateUpdate{Err: err} - } - - err = s.ClientTLSCreds.LoadNewTLSConfig(clientTLSConfig) - if err != nil { - log.WithError(err).Errorf("failed to update the client credentials") - updates <- CertificateUpdate{Err: err} - } - - // Update the external CA to use the new client TLS - // config using a copy without a serverName specified. - s.externalCA.UpdateTLSConfig(&tls.Config{ - Certificates: clientTLSConfig.Certificates, - RootCAs: clientTLSConfig.RootCAs, - MinVersion: tls.VersionTLS12, - }) - - err = s.ServerTLSCreds.LoadNewTLSConfig(serverTLSConfig) - if err != nil { - log.WithError(err).Errorf("failed to update the server TLS credentials") + // ignore errors - it will just try again laster + if err := RenewTLSConfigNow(ctx, s, remotes); err != nil { updates <- CertificateUpdate{Err: err} + } else { + updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()} } - - updates <- CertificateUpdate{Role: s.ClientTLSCreds.Role()} } }() @@ -447,13 +465,9 @@ func calculateRandomExpiry(validFrom, validUntil time.Time) time.Duration { // LoadTLSCreds loads tls credentials from the specified path and verifies that // thay are valid for the RootCA. -func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLSCreds, error) { +func LoadTLSCreds(rootCA RootCA, kr KeyReader) (*MutableTLSCreds, *MutableTLSCreds, error) { // Read both the Cert and Key from disk - cert, err := ioutil.ReadFile(paths.Cert) - if err != nil { - return nil, nil, err - } - key, err := ioutil.ReadFile(paths.Key) + cert, key, err := kr.Read() if err != nil { return nil, nil, err } @@ -482,24 +496,9 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS // Now that we know this certificate is valid, create a TLS Certificate for our // credentials - var ( - keyPair tls.Certificate - newErr error - ) - keyPair, err = tls.X509KeyPair(cert, key) + keyPair, err := tls.X509KeyPair(cert, key) if err != nil { - // This current keypair isn't valid. It's possible we crashed before we - // overwrote the current key. Let's try loading it from disk. - tempPaths := genTempPaths(paths) - key, newErr = ioutil.ReadFile(tempPaths.Key) - if newErr != nil { - return nil, nil, err - } - - keyPair, newErr = tls.X509KeyPair(cert, key) - if newErr != nil { - return nil, nil, err - } + return nil, nil, err } // Load the Certificates as server credentials @@ -519,13 +518,6 @@ func LoadTLSCreds(rootCA RootCA, paths CertPaths) (*MutableTLSCreds, *MutableTLS return clientTLSCreds, serverTLSCreds, nil } -func genTempPaths(path CertPaths) CertPaths { - return CertPaths{ - Key: filepath.Join(filepath.Dir(path.Key), "."+filepath.Base(path.Key)), - Cert: filepath.Join(filepath.Dir(path.Cert), "."+filepath.Base(path.Cert)), - } -} - // NewServerTLSConfig returns a tls.Config configured for a TLS Server, given a tls.Certificate // and the PEM-encoded root CA Certificate func NewServerTLSConfig(cert *tls.Certificate, rootCAPool *x509.CertPool) (*tls.Config, error) { diff --git a/ca/config_test.go b/ca/config_test.go index 2c7d2dd206..2ab9df8450 100644 --- a/ca/config_test.go +++ b/ca/config_test.go @@ -16,8 +16,61 @@ import ( "github.com/docker/swarmkit/ioutils" "github.com/docker/swarmkit/manager/state/store" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestDownloadRootCASuccess(t *testing.T) { + tc := testutils.NewTestCA(t) + defer tc.Stop() + + // Remove the CA cert + os.RemoveAll(tc.Paths.RootCA.Cert) + + rootCA, err := ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, tc.WorkerToken, tc.Remotes) + require.NoError(t, err) + require.NotNil(t, rootCA.Pool) + require.NotNil(t, rootCA.Cert) + require.Nil(t, rootCA.Signer) + require.False(t, rootCA.CanSign()) + require.Equal(t, tc.RootCA.Cert, rootCA.Cert) + + // Remove the CA cert + os.RemoveAll(tc.Paths.RootCA.Cert) + + // downloading without a join token also succeeds + rootCA, err = ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, "", tc.Remotes) + require.NoError(t, err) + require.NotNil(t, rootCA.Pool) + require.NotNil(t, rootCA.Cert) + require.Nil(t, rootCA.Signer) + require.False(t, rootCA.CanSign()) + require.Equal(t, tc.RootCA.Cert, rootCA.Cert) +} + +func TestDownloadRootCAWrongCAHash(t *testing.T) { + tc := testutils.NewTestCA(t) + defer tc.Stop() + + // Remove the CA cert + os.RemoveAll(tc.Paths.RootCA.Cert) + + // invalid token + _, err := ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, "invalidtoken", tc.Remotes) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid join token") + + // invalid hash token + splitToken := strings.Split(tc.ManagerToken, "-") + splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" + replacementToken := strings.Join(splitToken, "-") + + os.RemoveAll(tc.Paths.RootCA.Cert) + + _, err = ca.DownloadRootCA(tc.Context, tc.Paths.RootCA, replacementToken, tc.Remotes) + require.Error(t, err) + require.Contains(t, err.Error(), "remote CA does not match fingerprint.") +} + func TestLoadOrCreateSecurityConfigEmptyDir(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() @@ -25,15 +78,13 @@ func TestLoadOrCreateSecurityConfigEmptyDir(t *testing.T) { info := make(chan api.IssueNodeCertificateResponse, 1) // Remove all the contents from the temp dir and try again with a new node os.RemoveAll(tc.TempDir) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.Nil(t, nodeConfig.RootCA().Signer) - assert.False(t, nodeConfig.RootCA().CanSign()) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) assert.NotEmpty(t, <-info) } @@ -44,109 +95,27 @@ func TestLoadOrCreateSecurityConfigNoCerts(t *testing.T) { // Remove only the node certificates form the directory, and attest that we get // new certificates that are locally signed os.RemoveAll(tc.Paths.Node.Cert) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, tc.WorkerToken, ca.WorkerRole, tc.Remotes, nil) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, nil, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) - assert.True(t, nodeConfig.RootCA().CanSign()) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) info := make(chan api.IssueNodeCertificateResponse, 1) - // Remove only the node certificates form the directory, and attest that we get + // Remove only the node certificates form the directory, get a new rootCA, and attest that we get // new certificates that are issued by the remote CA - os.RemoveAll(tc.Paths.RootCA.Key) os.RemoveAll(tc.Paths.Node.Cert) - nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info) + rootCA, err := ca.GetLocalRootCA(tc.Paths.RootCA) assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.Nil(t, nodeConfig.RootCA().Signer) - assert.False(t, nodeConfig.RootCA().CanSign()) - assert.NotEmpty(t, <-info) -} - -func TestLoadOrCreateSecurityConfigWrongCAHash(t *testing.T) { - tc := testutils.NewTestCA(t) - defer tc.Stop() - - splitToken := strings.Split(tc.ManagerToken, "-") - splitToken[2] = "1kxftv4ofnc6mt30lmgipg6ngf9luhwqopfk1tz6bdmnkubg0e" - replacementToken := strings.Join(splitToken, "-") - - info := make(chan api.IssueNodeCertificateResponse, 1) - // Remove only the node certificates form the directory, and attest that we get - // new certificates that are issued by the remote CA - os.RemoveAll(tc.Paths.RootCA.Key) - os.RemoveAll(tc.Paths.RootCA.Cert) - os.RemoveAll(tc.Paths.Node.Cert) - _, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, replacementToken, ca.WorkerRole, tc.Remotes, info) - assert.Error(t, err) - assert.Contains(t, err.Error(), "remote CA does not match fingerprint.") -} - -func TestLoadOrCreateSecurityConfigInvalidCACert(t *testing.T) { - tc := testutils.NewTestCA(t) - defer tc.Stop() - - // First load the current nodeConfig. We'll verify that after we corrupt - // the certificate, another subsequent call with get us new certs - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) + nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, rootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, info, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - // We have a valid signer because we bootstrapped with valid root key-material - assert.NotNil(t, nodeConfig.RootCA().Signer) - assert.True(t, nodeConfig.RootCA().CanSign()) - - // Write some garbage to the CA cert - ioutil.WriteFile(tc.Paths.RootCA.Cert, []byte(`-----BEGIN CERTIFICATE-----\n -some random garbage\n ------END CERTIFICATE-----`), 0644) - - // We should get an error when the CA cert is invalid. - _, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.Error(t, err) - - // Not having a local cert should cause us to fallback to using the - // picker to get a remote. - assert.Nil(t, os.Remove(tc.Paths.RootCA.Cert)) - - // Validate we got a new valid state - newNodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) - assert.True(t, nodeConfig.RootCA().CanSign()) - - // Ensure that we have the same certificate as before - assert.Equal(t, nodeConfig.RootCA().Cert, newNodeConfig.RootCA().Cert) -} - -func TestLoadOrCreateSecurityConfigInvalidCAKey(t *testing.T) { - tc := testutils.NewTestCA(t) - defer tc.Stop() - - // Write some garbage to the root key - ioutil.WriteFile(tc.Paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n -some random garbage\n ------END EC PRIVATE KEY-----`), 0644) - - // We should get an error when the local ca private key is invalid. - _, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.Error(t, err) + assert.Equal(t, rootCA, *nodeConfig.RootCA()) + assert.NotEmpty(t, <-info) } func TestLoadOrCreateSecurityConfigInvalidCert(t *testing.T) { @@ -158,14 +127,13 @@ func TestLoadOrCreateSecurityConfigInvalidCert(t *testing.T) { some random garbage\n -----END CERTIFICATE-----`), 0644) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, "", ca.WorkerRole, tc.Remotes, nil, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) } func TestLoadOrCreateSecurityConfigInvalidKey(t *testing.T) { @@ -177,41 +145,27 @@ func TestLoadOrCreateSecurityConfigInvalidKey(t *testing.T) { some random garbage\n -----END EC PRIVATE KEY-----`), 0644) - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) + krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) + nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, "", ca.WorkerRole, tc.Remotes, nil, krw) assert.NoError(t, err) assert.NotNil(t, nodeConfig) assert.NotNil(t, nodeConfig.ClientTLSCreds) assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + assert.Equal(t, tc.RootCA, *nodeConfig.RootCA()) } -func TestLoadOrCreateSecurityConfigInvalidKeyWithValidTempKey(t *testing.T) { +func TestLoadOrCreateSecurityIncorrectPassphrase(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, tc.Remotes, nil) - assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + paths := ca.NewConfigPaths(tc.TempDir) + _, err := tc.RootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(paths.Node, []byte("kek"), nil), + "nodeID", ca.WorkerRole, tc.Organization) + require.NoError(t, err) - // Write some garbage to the Key - ioutil.WriteFile(tc.Paths.Node.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n -some random garbage\n ------END EC PRIVATE KEY-----`), 0644) - nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", ca.WorkerRole, nil, nil) - assert.NoError(t, err) - assert.NotNil(t, nodeConfig) - assert.NotNil(t, nodeConfig.ClientTLSCreds) - assert.NotNil(t, nodeConfig.ServerTLSCreds) - assert.NotNil(t, nodeConfig.RootCA().Pool) - assert.NotNil(t, nodeConfig.RootCA().Cert) - assert.NotNil(t, nodeConfig.RootCA().Signer) + _, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.RootCA, tc.WorkerToken, ca.WorkerRole, tc.Remotes, nil, + ca.NewKeyReadWriter(paths.Node, nil, nil)) + require.IsType(t, ca.ErrInvalidKEK{}, err) } func TestRenewTLSConfigWorker(t *testing.T) { @@ -240,7 +194,7 @@ func TestRenewTLSConfigWorker(t *testing.T) { }) // Create a new CSR and overwrite the key on disk - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, key, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issue a new certificate with the same details as the current config, but with 1 min expiration time @@ -253,8 +207,11 @@ func TestRenewTLSConfigWorker(t *testing.T) { err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644) assert.NoError(t, err) + err = ioutils.AtomicWriteFile(tc.Paths.Node.Key, key, 0600) + assert.NoError(t, err) + renew := make(chan struct{}) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) select { case <-time.After(10 * time.Second): assert.Fail(t, "TestRenewTLSConfig timed-out") @@ -291,7 +248,7 @@ func TestRenewTLSConfigManager(t *testing.T) { }) // Create a new CSR and overwrite the key on disk - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, key, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issue a new certificate with the same details as the current config, but with 1 min expiration time @@ -304,10 +261,13 @@ func TestRenewTLSConfigManager(t *testing.T) { err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644) assert.NoError(t, err) + err = ioutils.AtomicWriteFile(tc.Paths.Node.Key, key, 0600) + assert.NoError(t, err) + // Get a new nodeConfig with a TLS cert that has 1 minute to live renew := make(chan struct{}) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) select { case <-time.After(10 * time.Second): assert.Fail(t, "TestRenewTLSConfig timed-out") @@ -344,7 +304,7 @@ func TestRenewTLSConfigWithNoNode(t *testing.T) { }) // Create a new CSR and overwrite the key on disk - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, key, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issue a new certificate with the same details as the current config, but with 1 min expiration time @@ -357,6 +317,9 @@ func TestRenewTLSConfigWithNoNode(t *testing.T) { err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644) assert.NoError(t, err) + err = ioutils.AtomicWriteFile(tc.Paths.Node.Key, key, 0600) + assert.NoError(t, err) + // Delete the node from the backend store err = tc.MemoryStore.Update(func(tx store.Tx) error { node := store.GetNode(tx, nodeConfig.ClientTLSCreds.NodeID()) @@ -366,7 +329,7 @@ func TestRenewTLSConfigWithNoNode(t *testing.T) { assert.NoError(t, err) renew := make(chan struct{}) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) select { case <-time.After(10 * time.Second): assert.Fail(t, "TestRenewTLSConfig timed-out") @@ -390,7 +353,7 @@ func TestForceRenewTLSConfig(t *testing.T) { assert.NoError(t, err) renew := make(chan struct{}, 1) - updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Remotes, renew) + updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew) renew <- struct{}{} select { case <-time.After(10 * time.Second): diff --git a/ca/keyreadwriter.go b/ca/keyreadwriter.go new file mode 100644 index 0000000000..5cbb812671 --- /dev/null +++ b/ca/keyreadwriter.go @@ -0,0 +1,388 @@ +package ca + +import ( + "crypto/rand" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + + "crypto/tls" + + "github.com/docker/swarmkit/ioutils" + "github.com/pkg/errors" +) + +const ( + // keyPerms are the permissions used to write the TLS keys + keyPerms = 0600 + // certPerms are the permissions used to write TLS certificates + certPerms = 0644 + // versionHeader is the TLS PEM key header that contains the KEK version + versionHeader = "kek-version" +) + +// PEMKeyHeaders is something that needs to know about PEM headers when reading +// or writing TLS keys. +type PEMKeyHeaders interface { + // UnmarshalHeaders loads the headers map given the current KEK + UnmarshalHeaders(map[string]string, KEKData) (PEMKeyHeaders, error) + // MarshalHeaders returns a header map given the current KEK + MarshalHeaders(KEKData) (map[string]string, error) + // UpdateKEK may get a new PEMKeyHeaders if the KEK changes + UpdateKEK(KEKData, KEKData) PEMKeyHeaders +} + +// KeyReader reads a TLS cert and key from disk +type KeyReader interface { + Read() ([]byte, []byte, error) + Target() string +} + +// KeyWriter writes a TLS key and cert to disk +type KeyWriter interface { + Write([]byte, []byte, *KEKData) error + ViewAndUpdateHeaders(func(PEMKeyHeaders) (PEMKeyHeaders, error)) error + ViewAndRotateKEK(func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error + GetCurrentState() (PEMKeyHeaders, KEKData) + Target() string +} + +// KEKData provides an optional update to the kek when writing. The structure +// is needed so that we can tell the difference between "do not encrypt anymore" +// and there is "no update". +type KEKData struct { + KEK []byte + Version uint64 +} + +// ErrInvalidKEK means that we cannot decrypt the TLS key for some reason +type ErrInvalidKEK struct { + Wrapped error +} + +func (e ErrInvalidKEK) Error() string { + return e.Wrapped.Error() +} + +// KeyReadWriter is an object that knows how to read and write TLS keys and certs to disk, +// optionally encrypted and optionally updating PEM headers. +type KeyReadWriter struct { + mu sync.Mutex + kekData KEKData + paths CertPaths + headersObj PEMKeyHeaders +} + +// NewKeyReadWriter creates a new KeyReadWriter +func NewKeyReadWriter(paths CertPaths, kek []byte, headersObj PEMKeyHeaders) *KeyReadWriter { + return &KeyReadWriter{ + kekData: KEKData{KEK: kek}, + paths: paths, + headersObj: headersObj, + } +} + +// Migrate checks to see if a temporary key file exists. Older versions of +// swarmkit wrote temporary keys instead of temporary certificates, so +// migrate that temporary key if it exists. We want to write temporary certificates, +// instead of temporary keys, because we may need to periodically re-encrypt the +// keys and modify the headers, and it's easier to have a single canonical key +// location than two possible key locations. +func (k *KeyReadWriter) Migrate() error { + tmpPaths := k.genTempPaths() + keyBytes, err := ioutil.ReadFile(tmpPaths.Key) + if err != nil { + return nil // no key? no migration + } + + // it does exist - no need to decrypt, because previous versions of swarmkit + // which supported this temporary key did not support encrypting TLS keys + cert, err := ioutil.ReadFile(k.paths.Cert) + if err != nil { + return os.RemoveAll(tmpPaths.Key) // no cert? no migration + } + + // nope, this does not match the cert + if _, err = tls.X509KeyPair(cert, keyBytes); err != nil { + return os.RemoveAll(tmpPaths.Key) + } + + return os.Rename(tmpPaths.Key, k.paths.Key) +} + +// Read will read a TLS cert and key from the given paths +func (k *KeyReadWriter) Read() ([]byte, []byte, error) { + k.mu.Lock() + defer k.mu.Unlock() + keyBlock, err := k.readKey() + if err != nil { + return nil, nil, err + } + + if version, ok := keyBlock.Headers[versionHeader]; ok { + if versionInt, err := strconv.ParseUint(version, 10, 64); err == nil { + k.kekData.Version = versionInt + } + } + delete(keyBlock.Headers, versionHeader) + + if k.headersObj != nil { + newHeaders, err := k.headersObj.UnmarshalHeaders(keyBlock.Headers, k.kekData) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to read TLS key headers") + } + k.headersObj = newHeaders + } + + keyBytes := pem.EncodeToMemory(keyBlock) + cert, err := ioutil.ReadFile(k.paths.Cert) + // The cert is written to a temporary file first, then the key, and then + // the cert gets renamed - so, if interrupted, it's possible to end up with + // a cert that only exists in the temporary location. + switch { + case err == nil: + _, err = tls.X509KeyPair(cert, keyBytes) + case os.IsNotExist(err): //continue to try temp location + break + default: + return nil, nil, err + } + + // either the cert doesn't exist, or it doesn't match the key - try the temp file, if it exists + if err != nil { + var tempErr error + tmpPaths := k.genTempPaths() + cert, tempErr = ioutil.ReadFile(tmpPaths.Cert) + if tempErr != nil { + return nil, nil, err // return the original error + } + if _, tempErr := tls.X509KeyPair(cert, keyBytes); tempErr != nil { + os.RemoveAll(tmpPaths.Cert) // nope, it doesn't match either - remove and return the original error + return nil, nil, err + } + os.Rename(tmpPaths.Cert, k.paths.Cert) // try to move the temp cert back to the regular location + + } + + return cert, keyBytes, nil +} + +// ViewAndRotateKEK re-encrypts the key with a new KEK +func (k *KeyReadWriter) ViewAndRotateKEK(cb func(KEKData, PEMKeyHeaders) (KEKData, PEMKeyHeaders, error)) error { + k.mu.Lock() + defer k.mu.Unlock() + + updatedKEK, updatedHeaderObj, err := cb(k.kekData, k.headersObj) + if err != nil { + return err + } + + keyBlock, err := k.readKey() + if err != nil { + return err + } + + if err := k.writeKey(keyBlock, updatedKEK, updatedHeaderObj); err != nil { + return err + } + return nil +} + +// ViewAndUpdateHeaders updates the header manager, and updates any headers on the existing key +func (k *KeyReadWriter) ViewAndUpdateHeaders(cb func(PEMKeyHeaders) (PEMKeyHeaders, error)) error { + k.mu.Lock() + defer k.mu.Unlock() + + pkh, err := cb(k.headersObj) + if err != nil { + return err + } + + keyBlock, err := k.readKeyblock() + if err != nil { + return err + } + + headers := make(map[string]string) + if pkh != nil { + var err error + headers, err = pkh.MarshalHeaders(k.kekData) + if err != nil { + return err + } + } + // we WANT any original encryption headers + for key, value := range keyBlock.Headers { + normalizedKey := strings.TrimSpace(strings.ToLower(key)) + if normalizedKey == "proc-type" || normalizedKey == "dek-info" { + headers[key] = value + } + } + headers[versionHeader] = strconv.FormatUint(k.kekData.Version, 10) + keyBlock.Headers = headers + + if err = ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil { + return err + } + k.headersObj = pkh + return nil +} + +// GetCurrentState returns the current KEK data, including version +func (k *KeyReadWriter) GetCurrentState() (PEMKeyHeaders, KEKData) { + k.mu.Lock() + defer k.mu.Unlock() + return k.headersObj, k.kekData +} + +// Write attempts write a cert and key to text. This can also optionally update +// the KEK while writing, if an updated KEK is provided. If the pointer to the +// update KEK is nil, then we don't update. If the updated KEK itself is nil, +// then we update the KEK to be nil (data should be unencrypted). +func (k *KeyReadWriter) Write(certBytes, plaintextKeyBytes []byte, kekData *KEKData) error { + k.mu.Lock() + defer k.mu.Unlock() + + // current assumption is that the cert and key will be in the same directory + if err := os.MkdirAll(filepath.Dir(k.paths.Key), 0755); err != nil { + return err + } + + // Ensure that we will have a keypair on disk at all times by writing the cert to a + // temp path first. This is because we want to have only a single copy of the key + // for rotation and header modification. + tmpPaths := k.genTempPaths() + if err := ioutils.AtomicWriteFile(tmpPaths.Cert, certBytes, certPerms); err != nil { + return err + } + + keyBlock, _ := pem.Decode(plaintextKeyBytes) + if keyBlock == nil { + return errors.New("invalid PEM-encoded private key") + } + + if kekData == nil { + kekData = &k.kekData + } + pkh := k.headersObj + if k.headersObj != nil { + pkh = k.headersObj.UpdateKEK(k.kekData, *kekData) + } + + if err := k.writeKey(keyBlock, *kekData, pkh); err != nil { + return err + } + return os.Rename(tmpPaths.Cert, k.paths.Cert) +} + +func (k *KeyReadWriter) genTempPaths() CertPaths { + return CertPaths{ + Key: filepath.Join(filepath.Dir(k.paths.Key), "."+filepath.Base(k.paths.Key)), + Cert: filepath.Join(filepath.Dir(k.paths.Cert), "."+filepath.Base(k.paths.Cert)), + } +} + +// Target returns a string representation of this KeyReadWriter, namely where +// it is writing to +func (k *KeyReadWriter) Target() string { + return k.paths.Cert +} + +func (k *KeyReadWriter) readKeyblock() (*pem.Block, error) { + key, err := ioutil.ReadFile(k.paths.Key) + if err != nil { + return nil, err + } + + // Decode the PEM private key + keyBlock, _ := pem.Decode(key) + if keyBlock == nil { + return nil, errors.New("invalid PEM-encoded private key") + } + + return keyBlock, nil +} + +// readKey returns the decrypted key pem bytes, and enforces the KEK if applicable +// (writes it back with the correct encryption if it is not correctly encrypted) +func (k *KeyReadWriter) readKey() (*pem.Block, error) { + keyBlock, err := k.readKeyblock() + if err != nil { + return nil, err + } + + if !x509.IsEncryptedPEMBlock(keyBlock) { + return keyBlock, nil + } + + // If it's encrypted, we can't read without a passphrase (we're assuming + // empty passphrases iare invalid) + if k.kekData.KEK == nil { + return nil, ErrInvalidKEK{Wrapped: x509.IncorrectPasswordError} + } + + derBytes, err := x509.DecryptPEMBlock(keyBlock, k.kekData.KEK) + if err != nil { + return nil, ErrInvalidKEK{Wrapped: err} + } + // remove encryption PEM headers + headers := make(map[string]string) + mergePEMHeaders(headers, keyBlock.Headers) + + return &pem.Block{ + Type: keyBlock.Type, // the key type doesn't change + Bytes: derBytes, + Headers: headers, + }, nil +} + +// writeKey takes an unencrypted keyblock and, if the kek is not nil, encrypts it before +// writing it to disk. If the kek is nil, writes it to disk unencrypted. +func (k *KeyReadWriter) writeKey(keyBlock *pem.Block, kekData KEKData, pkh PEMKeyHeaders) error { + if kekData.KEK != nil { + encryptedPEMBlock, err := x509.EncryptPEMBlock(rand.Reader, + keyBlock.Type, + keyBlock.Bytes, + kekData.KEK, + x509.PEMCipherAES256) + if err != nil { + return err + } + if encryptedPEMBlock.Headers == nil { + return errors.New("unable to encrypt key - invalid PEM file produced") + } + keyBlock = encryptedPEMBlock + } + + if pkh != nil { + headers, err := pkh.MarshalHeaders(kekData) + if err != nil { + return err + } + mergePEMHeaders(keyBlock.Headers, headers) + } + keyBlock.Headers[versionHeader] = strconv.FormatUint(kekData.Version, 10) + + if err := ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil { + return err + } + k.kekData = kekData + k.headersObj = pkh + return nil +} + +// merges one set of PEM headers onto another, excepting for key encryption value +// "proc-type" and "dek-info" +func mergePEMHeaders(original, newSet map[string]string) { + for key, value := range newSet { + normalizedKey := strings.TrimSpace(strings.ToLower(key)) + if normalizedKey != "proc-type" && normalizedKey != "dek-info" { + original[key] = value + } + } +} diff --git a/ca/keyreadwriter_test.go b/ca/keyreadwriter_test.go new file mode 100644 index 0000000000..624adfef31 --- /dev/null +++ b/ca/keyreadwriter_test.go @@ -0,0 +1,414 @@ +package ca_test + +import ( + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/docker/swarmkit/ca" + "github.com/docker/swarmkit/ca/testutils" + "github.com/stretchr/testify/require" +) + +// can read and write tls keys that aren't encrypted, and that are encrypted. without +// a pem header manager, the headers are all preserved and not overwritten +func TestKeyReadWriter(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + expectedKey := key + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created + + checkCanReadWithKEK := func(kek []byte) *ca.KeyReadWriter { + k := ca.NewKeyReadWriter(path.Node, kek, nil) + readCert, readKey, err := k.Read() + require.NoError(t, err) + require.Equal(t, cert, readCert) + require.Equal(t, expectedKey, readKey, "Expected %s, Got %s", string(expectedKey), string(readKey)) + return k + } + + k := ca.NewKeyReadWriter(path.Node, nil, nil) + + // can't read things that don't exist + _, _, err = k.Read() + require.Error(t, err) + + // can write an unencrypted key with no updates + require.NoError(t, k.Write(cert, expectedKey, nil)) + + // can read unencrypted + k = checkCanReadWithKEK(nil) + _, kekData := k.GetCurrentState() + require.EqualValues(t, 0, kekData.Version) // the first version was 0 + + // write a key with headers to the key to make sure they're cleaned + keyBlock, _ := pem.Decode(expectedKey) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + expectedKey = pem.EncodeToMemory(keyBlock) + // write a version, but that's not what we'd expect back once we read + keyBlock.Headers["kek-version"] = "8" + require.NoError(t, ioutil.WriteFile(path.Node.Key, pem.EncodeToMemory(keyBlock), 0600)) + + // if a kek is provided, we can still read unencrypted keys, and read + // the provided version + k = checkCanReadWithKEK([]byte("original kek")) + _, kekData = k.GetCurrentState() + require.EqualValues(t, 8, kekData.Version) + + // we can update the kek and write at the same time + require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("new kek!"), Version: 3})) + + // the same kek can still read, and will continue to write with this key if + // no further kek updates are provided + _, _, err = k.Read() + require.NoError(t, err) + require.NoError(t, k.Write(cert, expectedKey, nil)) + + expectedKey = key + + // without the right kek, we can't read + k = ca.NewKeyReadWriter(path.Node, []byte("original kek"), nil) + _, _, err = k.Read() + require.Error(t, err) + + // same new key, just for sanity + k = checkCanReadWithKEK([]byte("new kek!")) + _, kekData = k.GetCurrentState() + require.EqualValues(t, 3, kekData.Version) + + // we can also change the kek back to nil, which means the key is unencrypted + require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: nil})) + k = checkCanReadWithKEK(nil) + _, kekData = k.GetCurrentState() + require.EqualValues(t, 0, kekData.Version) +} + +type testHeaders struct { + setHeaders func(map[string]string, ca.KEKData) (ca.PEMKeyHeaders, error) + newHeaders func(ca.KEKData) (map[string]string, error) +} + +func (p testHeaders) UnmarshalHeaders(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + if p.setHeaders != nil { + return p.setHeaders(h, k) + } + return nil, fmt.Errorf("set header error") +} + +func (p testHeaders) MarshalHeaders(k ca.KEKData) (map[string]string, error) { + if p.newHeaders != nil { + return p.newHeaders(k) + } + return nil, fmt.Errorf("update header error") +} + +func (p testHeaders) UpdateKEK(ca.KEKData, ca.KEKData) ca.PEMKeyHeaders { + return p +} + +// KeyReaderWriter makes a call to a get headers updater, if write is called, +// and set headers, if read is called. The KEK version header is always preserved +// no matter what. +func TestKeyReadWriterWithPemHeaderManager(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + // write a key with headers to the key to make sure it gets overwritten + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + key = pem.EncodeToMemory(keyBlock) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created + + // if if getting new headers fail, writing a key fails, and the key does not rotate + var count int + badKEKData := ca.KEKData{KEK: []byte("failed kek"), Version: 3} + k := ca.NewKeyReadWriter(path.Node, nil, testHeaders{newHeaders: func(k ca.KEKData) (map[string]string, error) { + if count == 0 { + count++ + require.Equal(t, badKEKData, k) + return nil, fmt.Errorf("fail") + } + require.Equal(t, ca.KEKData{}, k) + return nil, nil + }}) + // first write will fail + require.Error(t, k.Write(cert, key, &badKEKData)) + // the stored kek data will be not be updated because the write failed + _, kekData := k.GetCurrentState() + require.Equal(t, ca.KEKData{}, kekData) + // second write will succeed, using the original kek (nil) + require.NoError(t, k.Write(cert, key, nil)) + + var ( + headers map[string]string + kek ca.KEKData + ) + + // if setting headers fail, reading fails + k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(map[string]string, ca.KEKData) (ca.PEMKeyHeaders, error) { + return nil, fmt.Errorf("nope") + }}) + _, _, err = k.Read() + require.Error(t, err) + + k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + headers = h + kek = k + return testHeaders{}, nil + }}) + + _, _, err = k.Read() + require.NoError(t, err) + require.Equal(t, ca.KEKData{}, kek) + require.Equal(t, keyBlock.Headers, headers) + + // writing new headers is called with existing headers, and will write a key that has the headers + // returned by the header update function + k = ca.NewKeyReadWriter(path.Node, []byte("oldKek"), testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + require.Equal(t, []byte("newKEK"), kek.KEK) + return map[string]string{"updated": "headers"}, nil + }}) + require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("newKEK"), Version: 2})) + + // make sure headers were correctly set + k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + headers = h + kek = k + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) + require.Equal(t, ca.KEKData{KEK: []byte("newKEK"), Version: 2}, kek) + + _, kekData = k.GetCurrentState() + require.Equal(t, kek, kekData) + require.Equal(t, map[string]string{"updated": "headers"}, headers) +} + +func TestKeyReadWriterViewAndUpdateHeaders(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + + // write a key with headers to the key to make sure it gets passed when reading/writing headers + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + key = pem.EncodeToMemory(keyBlock) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644)) + require.NoError(t, ioutil.WriteFile(path.Node.Key, key, 0600)) + + // if the update headers callback function fails, updating headers fails + k := ca.NewKeyReadWriter(path.Node, nil, nil) + err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + require.Nil(t, h) + return nil, fmt.Errorf("nope") + }) + require.Error(t, err) + require.Equal(t, "nope", err.Error()) + + // updating headers succeed and is called with the latest kek data + err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + require.Nil(t, h) + return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + return map[string]string{"updated": "headers"}, nil + }}, nil + }) + require.NoError(t, err) + + k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + require.Equal(t, map[string]string{"updated": "headers"}, h) + require.Equal(t, ca.KEKData{}, k) + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) + + // we can also update headers on an encrypted key + k = ca.NewKeyReadWriter(path.Node, []byte("kek"), nil) + require.NoError(t, k.Write(cert, key, nil)) + + err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + require.Nil(t, h) + return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + require.Equal(t, ca.KEKData{KEK: []byte("kek")}, kek) + return map[string]string{"updated": "headers"}, nil + }}, nil + }) + require.NoError(t, err) + + k = ca.NewKeyReadWriter(path.Node, []byte("kek"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) { + require.Equal(t, map[string]string{"updated": "headers"}, h) + require.Equal(t, ca.KEKData{KEK: []byte("kek")}, k) + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) +} + +func TestKeyReadWriterViewAndRotateKEK(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + + // write a key with headers to the key to make sure it gets passed when reading/writing headers + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + keyBlock.Headers = map[string]string{"hello": "world"} + key = pem.EncodeToMemory(keyBlock) + require.NoError(t, ca.NewKeyReadWriter(path.Node, nil, nil).Write(cert, key, nil)) + + // if if getting new kek and headers fail, rotating a KEK fails, and the kek does not rotate + k := ca.NewKeyReadWriter(path.Node, nil, nil) + require.Error(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) { + require.Equal(t, ca.KEKData{}, k) + require.Nil(t, h) + return ca.KEKData{}, nil, fmt.Errorf("Nope") + })) + + // writing new headers will write a key that has the headers returned by the header update function + k = ca.NewKeyReadWriter(path.Node, []byte("oldKEK"), nil) + require.NoError(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) { + require.Equal(t, ca.KEKData{KEK: []byte("oldKEK")}, k) + require.Nil(t, h) + return ca.KEKData{KEK: []byte("newKEK"), Version: uint64(2)}, + testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) { + require.Equal(t, []byte("newKEK"), kek.KEK) + return map[string]string{"updated": "headers"}, nil + }}, nil + })) + + // ensure the key has been re-encrypted and we can read it + k = ca.NewKeyReadWriter(path.Node, nil, nil) + _, _, err = k.Read() + require.Error(t, err) + + var headers map[string]string + + k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, _ ca.KEKData) (ca.PEMKeyHeaders, error) { + headers = h + return testHeaders{}, nil + }}) + _, _, err = k.Read() + require.NoError(t, err) + require.Equal(t, map[string]string{"updated": "headers"}, headers) +} + +// If we abort in the middle of writing the key and cert, such that only the key is written +// to the final location, when we read we can still read the cert from the temporary +// location. +func TestTwoPhaseReadWrite(t *testing.T) { + cert1, _, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + cert2, key2, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + krw := ca.NewKeyReadWriter(path.Node, nil, nil) + + // put a directory in the location where the cert goes, so we can't actually move + // the cert from the temporary location to the final location. + require.NoError(t, os.Mkdir(filepath.Join(path.Node.Cert), 0755)) + require.Error(t, krw.Write(cert2, key2, nil)) + + // the temp cert file should exist + tempCertPath := filepath.Join(filepath.Dir(path.Node.Cert), "."+filepath.Base(path.Node.Cert)) + readCert, err := ioutil.ReadFile(tempCertPath) + require.NoError(t, err) + require.Equal(t, cert2, readCert) + + // remove the directory, to simulate it failing to write the first time + os.RemoveAll(path.Node.Cert) + readCert, readKey, err := krw.Read() + require.NoError(t, err) + require.Equal(t, cert2, readCert) + require.Equal(t, key2, readKey) + // the cert should have been moved to its proper location + _, err = os.Stat(tempCertPath) + require.True(t, os.IsNotExist(err)) + + // If the cert in the proper location doesn't match the key, the temp location is checked + require.NoError(t, ioutil.WriteFile(tempCertPath, cert2, 0644)) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert1, 0644)) + readCert, readKey, err = krw.Read() + require.NoError(t, err) + require.Equal(t, cert2, readCert) + require.Equal(t, key2, readKey) + // the cert should have been moved to its proper location + _, err = os.Stat(tempCertPath) + require.True(t, os.IsNotExist(err)) + + // If the cert in the temp location also doesn't match, the failure matching the + // correctly-located cert is returned + require.NoError(t, os.Remove(path.Node.Cert)) + require.NoError(t, ioutil.WriteFile(tempCertPath, cert1, 0644)) // mismatching cert + _, _, err = krw.Read() + require.True(t, os.IsNotExist(err)) + // the cert should have been removed + _, err = os.Stat(tempCertPath) + require.True(t, os.IsNotExist(err)) +} + +func TestKeyReadWriterMigrate(t *testing.T) { + cert, key, err := testutils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(filepath.Join(tempdir)) + + // if the key exists in an old location, migrate it from there. + tempKeyPath := filepath.Join(filepath.Dir(path.Node.Key), "."+filepath.Base(path.Node.Key)) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644)) + require.NoError(t, ioutil.WriteFile(tempKeyPath, key, 0600)) + + krw := ca.NewKeyReadWriter(path.Node, nil, nil) + require.NoError(t, krw.Migrate()) + _, err = os.Stat(tempKeyPath) + require.True(t, os.IsNotExist(err)) // it's been moved to the right place + _, _, err = krw.Read() + require.NoError(t, err) + + // migrate does not affect any existing files + dirList, err := ioutil.ReadDir(filepath.Dir(path.Node.Key)) + require.NoError(t, err) + require.NoError(t, krw.Migrate()) + dirList2, err := ioutil.ReadDir(filepath.Dir(path.Node.Key)) + require.NoError(t, err) + require.Equal(t, dirList, dirList2) + _, _, err = krw.Read() + require.NoError(t, err) +} diff --git a/ca/server_test.go b/ca/server_test.go index a4a01a6774..9a7ebd4dc8 100644 --- a/ca/server_test.go +++ b/ca/server_test.go @@ -14,6 +14,9 @@ import ( "github.com/stretchr/testify/require" ) +var _ api.CAServer = &ca.Server{} +var _ api.NodeCAServer = &ca.Server{} + func TestGetRootCACertificate(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() @@ -43,7 +46,7 @@ func TestIssueNodeCertificate(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Token: tc.WorkerToken} @@ -65,7 +68,7 @@ func TestForceRotationIsNoop(t *testing.T) { defer tc.Stop() // Get a new Certificate issued - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Token: tc.WorkerToken} @@ -111,7 +114,7 @@ func TestIssueNodeCertificateBrokenCA(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) tc.ExternalSigningServer.Flake() @@ -157,7 +160,7 @@ func TestIssueNodeCertificateWorkerRenewal(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) role := api.NodeRoleWorker @@ -179,7 +182,7 @@ func TestIssueNodeCertificateManagerRenewal(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) assert.NotNil(t, csr) @@ -202,7 +205,7 @@ func TestIssueNodeCertificateWorkerFromDifferentOrgRenewal(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) // Since we're using a client that has a different Organization, this request will be treated @@ -216,7 +219,7 @@ func TestNodeCertificateRenewalsDoNotRequireToken(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) role := api.NodeRoleManager @@ -254,7 +257,7 @@ func TestNewNodeCertificateRequiresToken(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issuance fails if no secret is provided @@ -333,7 +336,7 @@ func TestNewNodeCertificateBadToken(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node) + csr, _, err := ca.GenerateNewCSR() assert.NoError(t, err) // Issuance fails if wrong secret is provided diff --git a/ca/testutils/cautils.go b/ca/testutils/cautils.go index 089c7f6447..e7158a6598 100644 --- a/ca/testutils/cautils.go +++ b/ca/testutils/cautils.go @@ -46,6 +46,7 @@ type TestCA struct { WorkerToken string ManagerToken string Remotes remotes.Remotes + KeyReadWriter *ca.KeyReadWriter } // Stop cleansup after TestCA @@ -65,27 +66,27 @@ func (tc *TestCA) Stop() { // NewNodeConfig returns security config for a new node, given a role func (tc *TestCA) NewNodeConfig(role string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, tc.Organization, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, tc.Organization, tc.TempDir, withNonSigningRoot) } // WriteNewNodeConfig returns security config for a new node, given a role // saving the generated key and certificates to disk func (tc *TestCA) WriteNewNodeConfig(role string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, tc.Organization, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, tc.Organization, tc.TempDir, withNonSigningRoot) } // NewNodeConfigOrg returns security config for a new node, given a role and an org func (tc *TestCA) NewNodeConfigOrg(role, org string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, org, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, org, tc.TempDir, withNonSigningRoot) } // WriteNewNodeConfigOrg returns security config for a new node, given a role and an org // saving the generated key and certificates to disk func (tc *TestCA) WriteNewNodeConfigOrg(role, org string) (*ca.SecurityConfig, error) { withNonSigningRoot := tc.ExternalSigningServer != nil - return genSecurityConfig(tc.MemoryStore, tc.RootCA, role, org, tc.TempDir, withNonSigningRoot) + return genSecurityConfig(tc.MemoryStore, tc.RootCA, tc.KeyReadWriter, role, org, tc.TempDir, withNonSigningRoot) } // External controls whether or not NewTestCA() will create a TestCA server @@ -123,13 +124,15 @@ func NewTestCA(t *testing.T) *TestCA { } } - managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "", External) + krw := ca.NewKeyReadWriter(paths.Node, nil, nil) + + managerConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, organization, "", External) assert.NoError(t, err) - managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "", External) + managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, "swarm-test-org-2", "", External) assert.NoError(t, err) - workerConfig, err := genSecurityConfig(s, rootCA, ca.WorkerRole, organization, "", External) + workerConfig, err := genSecurityConfig(s, rootCA, krw, ca.WorkerRole, organization, "", External) assert.NoError(t, err) l, err := net.Listen("tcp", "127.0.0.1:0") @@ -194,6 +197,7 @@ func NewTestCA(t *testing.T) *TestCA { WorkerToken: workerToken, ManagerToken: managerToken, Remotes: remotes, + KeyReadWriter: krw, } } @@ -224,7 +228,7 @@ func createNode(s *store.MemoryStore, nodeID, role string, csr, cert []byte) err return err } -func genSecurityConfig(s *store.MemoryStore, rootCA ca.RootCA, role, org, tmpDir string, nonSigningRoot bool) (*ca.SecurityConfig, error) { +func genSecurityConfig(s *store.MemoryStore, rootCA ca.RootCA, krw *ca.KeyReadWriter, role, org, tmpDir string, nonSigningRoot bool) (*ca.SecurityConfig, error) { req := &cfcsr.CertificateRequest{ KeyRequest: cfcsr.NewBasicKeyRequest(), } @@ -297,7 +301,7 @@ func genSecurityConfig(s *store.MemoryStore, rootCA ca.RootCA, role, org, tmpDir } } - return ca.NewSecurityConfig(&rootCA, nodeClientTLSCreds, nodeServerTLSCreds), nil + return ca.NewSecurityConfig(&rootCA, krw, nodeClientTLSCreds, nodeServerTLSCreds), nil } func createClusterObject(t *testing.T, s *store.MemoryStore, clusterID, workerToken, managerToken string, externalCAs ...*api.ExternalCA) { @@ -323,9 +327,8 @@ func createClusterObject(t *testing.T, s *store.MemoryStore, clusterID, workerTo })) } -// createAndWriteRootCA creates a Certificate authority for a new Swarm Cluster. -// We're copying ca.CreateAndWriteRootCA, so we can have smaller key-sizes for tests -func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duration) (ca.RootCA, error) { +// CreateRootCertAndKey returns a generated certificate and key for a root CA +func CreateRootCertAndKey(rootCN string) ([]byte, []byte, error) { // Create a simple CSR for the CA using the default CA validator and policy req := cfcsr.CertificateRequest{ CN: rootCN, @@ -335,6 +338,13 @@ func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duratio // Generate the CA and get the certificate and private key cert, _, key, err := initca.New(&req) + return cert, key, err +} + +// createAndWriteRootCA creates a Certificate authority for a new Swarm Cluster. +// We're copying ca.CreateRootCA, so we can have smaller key-sizes for tests +func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duration) (ca.RootCA, error) { + cert, key, err := CreateRootCertAndKey(rootCN) if err != nil { return ca.RootCA{}, err } diff --git a/ca/testutils/externalutils.go b/ca/testutils/externalutils.go index 6b91bfd60b..8110e3dd60 100644 --- a/ca/testutils/externalutils.go +++ b/ca/testutils/externalutils.go @@ -34,7 +34,7 @@ func NewExternalSigningServer(rootCA ca.RootCA, basedir string) (*ExternalSignin Cert: filepath.Join(basedir, "server.crt"), Key: filepath.Join(basedir, "server.key"), } - serverCert, err := ca.GenerateAndSignNewTLSCert(rootCA, serverCN, serverOU, "", serverPaths) + serverCert, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(serverPaths, nil, nil), serverCN, serverOU, "") if err != nil { return nil, errors.Wrap(err, "unable to get TLS server certificate") } diff --git a/ca/transport_test.go b/ca/transport_test.go index 2d9279ba09..6b0b5ba2eb 100644 --- a/ca/transport_test.go +++ b/ca/transport_test.go @@ -13,7 +13,7 @@ func TestNewMutableTLS(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.ManagerRole, tc.Organization) + cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.ManagerRole, tc.Organization) assert.NoError(t, err) tlsConfig, err := ca.NewServerTLSConfig(cert, tc.RootCA.Pool) @@ -28,7 +28,7 @@ func TestGetAndValidateCertificateSubject(t *testing.T) { tc := testutils.NewTestCA(t) defer tc.Stop() - cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.ManagerRole, tc.Organization) + cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN", ca.ManagerRole, tc.Organization) assert.NoError(t, err) name, err := ca.GetAndValidateCertificateSubject([]tls.Certificate{*cert}) @@ -43,9 +43,9 @@ func TestLoadNewTLSConfig(t *testing.T) { defer tc.Stop() // Create two different certs and two different TLS configs - cert1, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN1", ca.ManagerRole, tc.Organization) + cert1, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN1", ca.ManagerRole, tc.Organization) assert.NoError(t, err) - cert2, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN2", ca.WorkerRole, tc.Organization) + cert2, err := tc.RootCA.IssueAndSaveNewCertificates(tc.KeyReadWriter, "CN2", ca.WorkerRole, tc.Organization) assert.NoError(t, err) tlsConfig1, err := ca.NewServerTLSConfig(cert1, tc.RootCA.Pool) assert.NoError(t, err) diff --git a/cmd/external-ca-example/main.go b/cmd/external-ca-example/main.go index 2037416404..f837422441 100644 --- a/cmd/external-ca-example/main.go +++ b/cmd/external-ca-example/main.go @@ -21,7 +21,7 @@ func main() { } // Initialize the Root CA. - rootCA, err := ca.CreateAndWriteRootCA("external-ca-example", rootPaths) + rootCA, err := ca.CreateRootCA("external-ca-example", rootPaths) if err != nil { logrus.Fatalf("unable to initialize Root CA: %s", err) } @@ -31,7 +31,9 @@ func main() { clusterID := identity.NewID() nodeID := identity.NewID() - if _, err := ca.GenerateAndSignNewTLSCert(rootCA, nodeID, ca.ManagerRole, clusterID, nodeConfigPaths.Node); err != nil { + + kw := ca.NewKeyReadWriter(nodeConfigPaths.Node, nil, nil) + if _, err := rootCA.IssueAndSaveNewCertificates(kw, nodeID, ca.ManagerRole, clusterID); err != nil { logrus.Fatalf("unable to create initial manager node credentials: %s", err) } diff --git a/node/node.go b/node/node.go index 53195d7a90..606c41c342 100644 --- a/node/node.go +++ b/node/node.go @@ -21,6 +21,7 @@ import ( "github.com/docker/swarmkit/ioutils" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/remotes" "github.com/docker/swarmkit/xnet" "github.com/pkg/errors" @@ -34,6 +35,10 @@ const stateFilename = "state.json" var ( errNodeStarted = errors.New("node: already started") errNodeNotStarted = errors.New("node: not started") + certDirectory = "certificates" + + // ErrInvalidUnlockKey is returned when we can't decrypt the TLS certificate + ErrInvalidUnlockKey = errors.New("node is locked, and needs a valid unlock key") ) // Config provides values for a Node. @@ -81,6 +86,14 @@ type Config struct { // HeartbeatTick defines the amount of ticks between each // heartbeat sent to other members for health-check purposes HeartbeatTick uint32 + + // AutoLockManagers determines whether or not an unlock key will be generated + // when bootstrapping a new cluster for the first time + AutoLockManagers bool + + // UnlockKey is the key to unlock a node - used for decrypting at rest. This + // only applies to nodes that have already joined a cluster. + UnlockKey []byte } // Node implements the primary node functionality for a member of a swarm @@ -106,6 +119,7 @@ type Node struct { agent *agent.Agent manager *manager.Manager notifyNodeChange chan *api.Node // used to send role updates from the dispatcher api on promotion/demotion + unlockKey []byte } // RemoteAPIAddr returns address on which remote manager api listens. @@ -150,12 +164,18 @@ func New(c *Config) (*Node, error) { ready: make(chan struct{}), certificateRequested: make(chan struct{}), notifyNodeChange: make(chan *api.Node, 1), + unlockKey: c.UnlockKey, } + + if n.config.JoinAddr != "" || n.config.ForceNewCluster { + n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename)) + if n.config.JoinAddr != "" { + n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight) + } + } + n.roleCond = sync.NewCond(n.RLocker()) n.connCond = sync.NewCond(n.RLocker()) - if err := n.loadCertificates(); err != nil { - return nil, err - } return n, nil } @@ -189,46 +209,7 @@ func (n *Node) run(ctx context.Context) (err error) { } }() - // NOTE: When this node is created by NewNode(), our nodeID is set if - // n.loadCertificates() succeeded in loading TLS credentials. - if n.config.JoinAddr == "" && n.nodeID == "" { - if err := n.bootstrapCA(); err != nil { - return err - } - } - - if n.config.JoinAddr != "" || n.config.ForceNewCluster { - n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename)) - if n.config.JoinAddr != "" { - n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, remotes.DefaultObservationWeight) - } - } - - // Obtain new certs and setup TLS certificates renewal for this node: - // - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued - // - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows - // us to display the ID before the certificate gets issued (for potential approval). - // - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate. - // - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay - // up to date. - issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1) - go func() { - select { - case <-ctx.Done(): - case resp := <-issueResponseChan: - log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{ - "node.id": resp.NodeID, - }).Debugf("requesting certificate") - n.Lock() - n.nodeID = resp.NodeID - n.nodeMembership = resp.NodeMembership - n.Unlock() - close(n.certificateRequested) - } - }() - - certDir := filepath.Join(n.config.StateDir, "certificates") - securityConfig, err := ca.LoadOrCreateSecurityConfig(ctx, certDir, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan) + securityConfig, err := n.loadSecurityConfig(ctx) if err != nil { return err } @@ -244,10 +225,6 @@ func (n *Node) run(ctx context.Context) (err error) { } defer db.Close() - if err := n.loadCertificates(); err != nil { - return err - } - forceCertRenewal := make(chan struct{}) renewCert := func() { select { @@ -289,7 +266,7 @@ func (n *Node) run(ctx context.Context) (err error) { } }() - updates := ca.RenewTLSConfig(ctx, securityConfig, certDir, n.remotes, forceCertRenewal) + updates := ca.RenewTLSConfig(ctx, securityConfig, n.remotes, forceCertRenewal) go func() { for { select { @@ -515,40 +492,101 @@ func (n *Node) Remotes() []api.Peer { return remotes } -func (n *Node) loadCertificates() error { - certDir := filepath.Join(n.config.StateDir, "certificates") - rootCA, err := ca.GetLocalRootCA(certDir) - if err != nil { - if err == ca.ErrNoLocalRootCA { - return nil +func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) { + paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory)) + var securityConfig *ca.SecurityConfig + + krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, nil) + if err := krw.Migrate(); err != nil { + return nil, err + } + + // Check if we already have a valid certificates on disk. + rootCA, err := ca.GetLocalRootCA(paths.RootCA) + if err != nil && err != ca.ErrNoLocalRootCA { + return nil, err + } + if err == nil { + clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw) + _, ok := errors.Cause(err).(ca.ErrInvalidKEK) + switch { + case err == nil: + securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds) + log.G(ctx).Debug("loaded CA and TLS certificates") + case ok: + return nil, ErrInvalidUnlockKey + case os.IsNotExist(err): + break + default: + return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert) } - return err } - configPaths := ca.NewConfigPaths(certDir) - clientTLSCreds, _, err := ca.LoadTLSCreds(rootCA, configPaths.Node) - if err != nil { - if os.IsNotExist(err) { - return nil + + if securityConfig == nil { + switch { + case n.config.JoinAddr == "": + // if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key + n.unlockKey = nil + if n.config.AutoLockManagers { + n.unlockKey = encryption.GenerateSecretKey() + } + krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, nil) + rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + if err != nil { + return nil, err + } + log.G(ctx).Debug("generated CA key and certificate") + case err == ca.ErrNoLocalRootCA: + rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes) + if err != nil { + return nil, err + } + log.G(ctx).Debug("downloaded CA certificate") } - return errors.Wrapf(err, "error while loading TLS Certificate in %s", configPaths.Node.Cert) + // Obtain new certs and setup TLS certificates renewal for this node: + // - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued + // - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows + // us to display the ID before the certificate gets issued (for potential approval). + // - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate. + // - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay + // up to date. + issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1) + go func() { + select { + case <-ctx.Done(): + case resp := <-issueResponseChan: + log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{ + "node.id": resp.NodeID, + }).Debugf("loaded TLS certificate") + n.Lock() + n.nodeID = resp.NodeID + n.nodeMembership = resp.NodeMembership + n.Unlock() + close(n.certificateRequested) + } + }() + + // LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS + // certificates and write them to disk + securityConfig, err = ca.LoadOrCreateSecurityConfig( + ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw) + if err != nil { + if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok { + return nil, ErrInvalidUnlockKey + } + return nil, err + } } - // todo: try csr if no cert or store nodeID/role in some other way + n.Lock() - n.role = clientTLSCreds.Role() - n.nodeID = clientTLSCreds.NodeID() + n.role = securityConfig.ClientTLSCreds.Role() + n.nodeID = securityConfig.ClientTLSCreds.NodeID() n.nodeMembership = api.NodeMembershipAccepted n.roleCond.Broadcast() n.Unlock() - return nil -} - -func (n *Node) bootstrapCA() error { - if err := ca.BootstrapCluster(filepath.Join(n.config.StateDir, "certificates")); err != nil { - return err - } - return n.loadCertificates() + return securityConfig, nil } func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error { diff --git a/node/node_test.go b/node/node_test.go new file mode 100644 index 0000000000..c3caa279ef --- /dev/null +++ b/node/node_test.go @@ -0,0 +1,207 @@ +package node + +import ( + "context" + "crypto/x509" + "encoding/pem" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/docker/swarmkit/ca" + cautils "github.com/docker/swarmkit/ca/testutils" + "github.com/docker/swarmkit/identity" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +// If there is nothing on disk and no join addr, we create a new CA and a new set of TLS certs. +// If AutoLockManagers is enabled, the TLS key is encrypted with a randomly generated lock key. +func TestLoadSecurityConfigNewNode(t *testing.T) { + for _, autoLockManagers := range []bool{true, false} { + tempdir, err := ioutil.TempDir("", "test-new-node") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + + node, err := New(&Config{ + StateDir: tempdir, + AutoLockManagers: autoLockManagers, + }) + require.NoError(t, err) + securityConfig, err := node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + unencryptedReader := ca.NewKeyReadWriter(paths.Node, nil, nil) + _, _, err = unencryptedReader.Read() + if !autoLockManagers { + require.NoError(t, err) + } else { + require.IsType(t, ca.ErrInvalidKEK{}, err) + } + } +} + +// If there's only a root CA on disk (no TLS certs), and no join addr, we create a new CA +// and a new set of TLS certs. Similarly if there's only a TLS cert and key, and no CA. +func TestLoadSecurityConfigPartialCertsOnDisk(t *testing.T) { + tempdir, err := ioutil.TempDir("", "test-new-node") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + require.NoError(t, err) + + node, err := New(&Config{ + StateDir: tempdir, + }) + require.NoError(t, err) + securityConfig, err := node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + cert, key, err := securityConfig.KeyReader().Read() + require.NoError(t, err) + + // a new CA was generated because no existing TLS certs were present + require.NotEqual(t, rootCA.Cert, securityConfig.RootCA().Cert) + + // if the TLS key and cert are on disk, but there's no CA, a new CA and TLS + // key+cert are generated + require.NoError(t, os.RemoveAll(paths.RootCA.Cert)) + + node, err = New(&Config{ + StateDir: tempdir, + }) + require.NoError(t, err) + securityConfig, err = node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + newCert, newKey, err := securityConfig.KeyReader().Read() + require.NoError(t, err) + require.NotEqual(t, cert, newCert) + require.NotEqual(t, key, newKey) +} + +// If there are CAs and TLS certs on disk, it tries to load and fails if there +// are any errors, even if a join token is provided. +func TestLoadSecurityConfigLoadFromDisk(t *testing.T) { + tempdir, err := ioutil.TempDir("", "test-load-node-tls") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + + tc := cautils.NewTestCA(t) + defer tc.Stop() + peer, err := tc.Remotes.Select() + require.NoError(t, err) + + // Load successfully with valid passphrase + rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + require.NoError(t, err) + krw := ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil) + require.NoError(t, err) + _, err = rootCA.IssueAndSaveNewCertificates(krw, identity.NewID(), ca.WorkerRole, identity.NewID()) + require.NoError(t, err) + + node, err := New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + UnlockKey: []byte("passphrase"), + }) + require.NoError(t, err) + securityConfig, err := node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + require.NotNil(t, securityConfig) + + // Invalid passphrase + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.Equal(t, ErrInvalidUnlockKey, err) + + // Invalid CA + rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) + require.NoError(t, err) + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + UnlockKey: []byte("passphrase"), + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.IsType(t, x509.UnknownAuthorityError{}, errors.Cause(err)) +} + +// If there is no CA, and a join addr is provided, one is downloaded from the +// join server. If there is a CA, it is just loaded from disk. The TLS key and +// cert are also downloaded. +func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) { + tempdir, err := ioutil.TempDir("", "test-join-node") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates")) + + // join addr is invalid + node, err := New(&Config{ + StateDir: tempdir, + JoinAddr: "127.0.0.1:12", + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.Error(t, err) + + tc := cautils.NewTestCA(t) + defer tc.Stop() + + peer, err := tc.Remotes.Select() + require.NoError(t, err) + + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + + // remove the TLS cert and key, and mark the root CA cert so that we will + // know if it gets replaced + require.NoError(t, os.Remove(paths.Node.Cert)) + require.NoError(t, os.Remove(paths.Node.Key)) + certBytes, err := ioutil.ReadFile(paths.RootCA.Cert) + require.NoError(t, err) + pemBlock, _ := pem.Decode(certBytes) + require.NotNil(t, pemBlock) + pemBlock.Headers["marked"] = "true" + certBytes = pem.EncodeToMemory(pemBlock) + require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, certBytes, 0644)) + + node, err = New(&Config{ + StateDir: tempdir, + JoinAddr: peer.Addr, + JoinToken: tc.ManagerToken, + }) + require.NoError(t, err) + _, err = node.loadSecurityConfig(context.Background()) + require.NoError(t, err) + + // make sure the CA cert has not been replaced + readCertBytes, err := ioutil.ReadFile(paths.RootCA.Cert) + require.NoError(t, err) + require.Equal(t, certBytes, readCertBytes) +} From 116ceb7ecd80fa795f973698c5f56700f160ce88 Mon Sep 17 00:00:00 2001 From: cyli Date: Fri, 21 Oct 2016 22:32:10 -0700 Subject: [PATCH 2/3] Provide a mTLS CA server endpoint to get the latest manager unlock key so that so that TLS certificates are never written to disk after they are downloaded from the swarm CA. Also, update the cluster spec and object so that a user can enable and disable autolocking managers. If autolocking is enabled, a shared key will be used by all the managers to encrypt TLS keys and raft DEKs. Also, provide the control API and swarmd/swarmct commands to enable/disable autolocking, and rotate lock keys. Signed-off-by: cyli --- api/ca.pb.go | 456 +++++++++++++++-- api/ca.proto | 12 + api/control.pb.go | 347 +++++++------ api/control.proto | 19 +- api/objects.pb.go | 219 ++++++--- api/objects.proto | 7 + api/specs.pb.go | 255 ++++++---- api/specs.proto | 3 + api/types.pb.go | 678 ++++++++++++++++---------- api/types.proto | 7 + ca/certificates.go | 92 +++- ca/certificates_test.go | 42 +- ca/server.go | 27 + ca/server_test.go | 51 ++ cmd/swarmctl/cluster/cmd.go | 1 + cmd/swarmctl/cluster/unlockkey.go | 50 ++ cmd/swarmctl/cluster/update.go | 27 +- cmd/swarmd/main.go | 22 + manager/controlapi/cluster.go | 31 +- manager/controlapi/cluster_test.go | 144 +++++- manager/encryption/encryption.go | 4 +- manager/encryption/encryption_test.go | 2 +- manager/manager.go | 44 +- manager/manager_test.go | 26 +- node/node.go | 16 +- 25 files changed, 1891 insertions(+), 691 deletions(-) create mode 100644 cmd/swarmctl/cluster/unlockkey.go diff --git a/api/ca.pb.go b/api/ca.pb.go index 3262fe4f66..19d103e84e 100644 --- a/api/ca.pb.go +++ b/api/ca.pb.go @@ -89,6 +89,22 @@ func (m *GetRootCACertificateResponse) Reset() { *m = GetRoot func (*GetRootCACertificateResponse) ProtoMessage() {} func (*GetRootCACertificateResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{5} } +type GetUnlockKeyRequest struct { +} + +func (m *GetUnlockKeyRequest) Reset() { *m = GetUnlockKeyRequest{} } +func (*GetUnlockKeyRequest) ProtoMessage() {} +func (*GetUnlockKeyRequest) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{6} } + +type GetUnlockKeyResponse struct { + UnlockKey []byte `protobuf:"bytes,1,opt,name=unlock_key,json=unlockKey,proto3" json:"unlock_key,omitempty"` + Version Version `protobuf:"bytes,2,opt,name=version" json:"version"` +} + +func (m *GetUnlockKeyResponse) Reset() { *m = GetUnlockKeyResponse{} } +func (*GetUnlockKeyResponse) ProtoMessage() {} +func (*GetUnlockKeyResponse) Descriptor() ([]byte, []int) { return fileDescriptorCa, []int{7} } + func init() { proto.RegisterType((*NodeCertificateStatusRequest)(nil), "docker.swarmkit.v1.NodeCertificateStatusRequest") proto.RegisterType((*NodeCertificateStatusResponse)(nil), "docker.swarmkit.v1.NodeCertificateStatusResponse") @@ -96,6 +112,8 @@ func init() { proto.RegisterType((*IssueNodeCertificateResponse)(nil), "docker.swarmkit.v1.IssueNodeCertificateResponse") proto.RegisterType((*GetRootCACertificateRequest)(nil), "docker.swarmkit.v1.GetRootCACertificateRequest") proto.RegisterType((*GetRootCACertificateResponse)(nil), "docker.swarmkit.v1.GetRootCACertificateResponse") + proto.RegisterType((*GetUnlockKeyRequest)(nil), "docker.swarmkit.v1.GetUnlockKeyRequest") + proto.RegisterType((*GetUnlockKeyResponse)(nil), "docker.swarmkit.v1.GetUnlockKeyResponse") } type authenticatedWrapperCAServer struct { @@ -115,6 +133,14 @@ func (p *authenticatedWrapperCAServer) GetRootCACertificate(ctx context.Context, return p.local.GetRootCACertificate(ctx, r) } +func (p *authenticatedWrapperCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) { + + if err := p.authorize(ctx, []string{"swarm-manager"}); err != nil { + return nil, err + } + return p.local.GetUnlockKey(ctx, r) +} + type authenticatedWrapperNodeCAServer struct { local NodeCAServer authorize func(context.Context, []string) error @@ -211,6 +237,29 @@ func (m *GetRootCACertificateResponse) Copy() *GetRootCACertificateResponse { return o } +func (m *GetUnlockKeyRequest) Copy() *GetUnlockKeyRequest { + if m == nil { + return nil + } + + o := &GetUnlockKeyRequest{} + + return o +} + +func (m *GetUnlockKeyResponse) Copy() *GetUnlockKeyResponse { + if m == nil { + return nil + } + + o := &GetUnlockKeyResponse{ + UnlockKey: m.UnlockKey, + Version: *m.Version.Copy(), + } + + return o +} + func (this *NodeCertificateStatusRequest) GoString() string { if this == nil { return "nil" @@ -278,6 +327,26 @@ func (this *GetRootCACertificateResponse) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *GetUnlockKeyRequest) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 4) + s = append(s, "&api.GetUnlockKeyRequest{") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *GetUnlockKeyResponse) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 6) + s = append(s, "&api.GetUnlockKeyResponse{") + s = append(s, "UnlockKey: "+fmt.Sprintf("%#v", this.UnlockKey)+",\n") + s = append(s, "Version: "+strings.Replace(this.Version.GoString(), `&`, ``, 1)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func valueToGoStringCa(v interface{}, typ string) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -317,6 +386,9 @@ const _ = grpc.SupportPackageIsVersion3 type CAClient interface { GetRootCACertificate(ctx context.Context, in *GetRootCACertificateRequest, opts ...grpc.CallOption) (*GetRootCACertificateResponse, error) + // GetUnlockKey returns the current unlock key for the cluster for the role of the client + // asking. + GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error) } type cAClient struct { @@ -336,10 +408,22 @@ func (c *cAClient) GetRootCACertificate(ctx context.Context, in *GetRootCACertif return out, nil } +func (c *cAClient) GetUnlockKey(ctx context.Context, in *GetUnlockKeyRequest, opts ...grpc.CallOption) (*GetUnlockKeyResponse, error) { + out := new(GetUnlockKeyResponse) + err := grpc.Invoke(ctx, "/docker.swarmkit.v1.CA/GetUnlockKey", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for CA service type CAServer interface { GetRootCACertificate(context.Context, *GetRootCACertificateRequest) (*GetRootCACertificateResponse, error) + // GetUnlockKey returns the current unlock key for the cluster for the role of the client + // asking. + GetUnlockKey(context.Context, *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) } func RegisterCAServer(s *grpc.Server, srv CAServer) { @@ -364,6 +448,24 @@ func _CA_GetRootCACertificate_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _CA_GetUnlockKey_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetUnlockKeyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CAServer).GetUnlockKey(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/docker.swarmkit.v1.CA/GetUnlockKey", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CAServer).GetUnlockKey(ctx, req.(*GetUnlockKeyRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _CA_serviceDesc = grpc.ServiceDesc{ ServiceName: "docker.swarmkit.v1.CA", HandlerType: (*CAServer)(nil), @@ -372,6 +474,10 @@ var _CA_serviceDesc = grpc.ServiceDesc{ MethodName: "GetRootCACertificate", Handler: _CA_GetRootCACertificate_Handler, }, + { + MethodName: "GetUnlockKey", + Handler: _CA_GetUnlockKey_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: fileDescriptorCa, @@ -642,6 +748,56 @@ func (m *GetRootCACertificateResponse) MarshalTo(data []byte) (int, error) { return i, nil } +func (m *GetUnlockKeyRequest) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *GetUnlockKeyRequest) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + return i, nil +} + +func (m *GetUnlockKeyResponse) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *GetUnlockKeyResponse) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.UnlockKey) > 0 { + data[i] = 0xa + i++ + i = encodeVarintCa(data, i, uint64(len(m.UnlockKey))) + i += copy(data[i:], m.UnlockKey) + } + data[i] = 0x12 + i++ + i = encodeVarintCa(data, i, uint64(m.Version.Size())) + n3, err := m.Version.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n3 + return i, nil +} + func encodeFixed64Ca(data []byte, offset int, v uint64) int { data[offset] = uint8(v) data[offset+1] = uint8(v >> 8) @@ -767,6 +923,37 @@ func (p *raftProxyCAServer) GetRootCACertificate(ctx context.Context, r *GetRoot return resp, err } +func (p *raftProxyCAServer) GetUnlockKey(ctx context.Context, r *GetUnlockKeyRequest) (*GetUnlockKeyResponse, error) { + + conn, err := p.connSelector.LeaderConn(ctx) + if err != nil { + if err == raftselector.ErrIsLeader { + return p.local.GetUnlockKey(ctx, r) + } + return nil, err + } + modCtx, err := p.runCtxMods(ctx) + if err != nil { + return nil, err + } + + resp, err := NewCAClient(conn).GetUnlockKey(modCtx, r) + if err != nil { + if !strings.Contains(err.Error(), "is closing") && !strings.Contains(err.Error(), "the connection is unavailable") && !strings.Contains(err.Error(), "connection error") { + return resp, err + } + conn, err := p.pollNewLeaderConn(ctx) + if err != nil { + if err == raftselector.ErrIsLeader { + return p.local.GetUnlockKey(ctx, r) + } + return nil, err + } + return NewCAClient(conn).GetUnlockKey(modCtx, r) + } + return resp, err +} + type raftProxyNodeCAServer struct { local NodeCAServer connSelector raftselector.ConnProvider @@ -965,6 +1152,24 @@ func (m *GetRootCACertificateResponse) Size() (n int) { return n } +func (m *GetUnlockKeyRequest) Size() (n int) { + var l int + _ = l + return n +} + +func (m *GetUnlockKeyResponse) Size() (n int) { + var l int + _ = l + l = len(m.UnlockKey) + if l > 0 { + n += 1 + l + sovCa(uint64(l)) + } + l = m.Version.Size() + n += 1 + l + sovCa(uint64(l)) + return n +} + func sovCa(x uint64) (n int) { for { n++ @@ -1041,6 +1246,26 @@ func (this *GetRootCACertificateResponse) String() string { }, "") return s } +func (this *GetUnlockKeyRequest) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&GetUnlockKeyRequest{`, + `}`, + }, "") + return s +} +func (this *GetUnlockKeyResponse) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&GetUnlockKeyResponse{`, + `UnlockKey:` + fmt.Sprintf("%v", this.UnlockKey) + `,`, + `Version:` + strings.Replace(strings.Replace(this.Version.String(), "Version", "Version", 1), `&`, ``, 1) + `,`, + `}`, + }, "") + return s +} func valueToStringCa(v interface{}) string { rv := reflect.ValueOf(v) if rv.IsNil() { @@ -1602,6 +1827,167 @@ func (m *GetRootCACertificateResponse) Unmarshal(data []byte) error { } return nil } +func (m *GetUnlockKeyRequest) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetUnlockKeyRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetUnlockKeyRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := skipCa(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCa + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *GetUnlockKeyResponse) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: GetUnlockKeyResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: GetUnlockKeyResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UnlockKey", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthCa + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UnlockKey = append(m.UnlockKey[:0], data[iNdEx:postIndex]...) + if m.UnlockKey == nil { + m.UnlockKey = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCa + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthCa + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCa(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCa + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipCa(data []byte) (n int, err error) { l := len(data) iNdEx := 0 @@ -1710,36 +2096,42 @@ var ( func init() { proto.RegisterFile("ca.proto", fileDescriptorCa) } var fileDescriptorCa = []byte{ - // 493 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x94, 0xcf, 0x6e, 0xd3, 0x40, - 0x10, 0xc6, 0xbb, 0x0e, 0xa4, 0x65, 0x52, 0x05, 0xb4, 0x04, 0x29, 0xa4, 0xa9, 0x53, 0x99, 0x03, - 0x9c, 0x9c, 0xd6, 0x70, 0xe2, 0x44, 0x62, 0x24, 0x94, 0x03, 0x08, 0x6d, 0x1e, 0x00, 0xb9, 0xf6, - 0x10, 0xac, 0x24, 0x5e, 0xe3, 0xdd, 0x80, 0xb8, 0x21, 0x81, 0x38, 0x70, 0x47, 0x70, 0xe2, 0x11, - 0x78, 0x8e, 0x8a, 0x13, 0x47, 0x4e, 0x15, 0xf1, 0x03, 0x20, 0x1e, 0x01, 0xed, 0xda, 0x21, 0xfd, - 0xb3, 0x89, 0xca, 0xc9, 0x3b, 0xb3, 0xf3, 0x7d, 0xfe, 0xed, 0x8c, 0xd7, 0xb0, 0x15, 0x06, 0x6e, - 0x9a, 0x71, 0xc9, 0x29, 0x8d, 0x78, 0x38, 0xc6, 0xcc, 0x15, 0xaf, 0x83, 0x6c, 0x3a, 0x8e, 0xa5, - 0xfb, 0xea, 0xa0, 0x55, 0x93, 0x6f, 0x52, 0x14, 0x45, 0x41, 0xab, 0x26, 0x52, 0x0c, 0x17, 0x41, - 0x63, 0xc4, 0x47, 0x5c, 0x2f, 0xbb, 0x6a, 0x55, 0x66, 0xaf, 0xa7, 0x93, 0xd9, 0x28, 0x4e, 0xba, - 0xc5, 0xa3, 0x48, 0x3a, 0x3e, 0xb4, 0x9f, 0xf0, 0x08, 0x7d, 0xcc, 0x64, 0xfc, 0x3c, 0x0e, 0x03, - 0x89, 0x43, 0x19, 0xc8, 0x99, 0x60, 0xf8, 0x72, 0x86, 0x42, 0xd2, 0x5b, 0xb0, 0x99, 0xf0, 0x08, - 0x9f, 0xc5, 0x51, 0x93, 0xec, 0x91, 0x3b, 0x57, 0xfa, 0x90, 0x1f, 0x77, 0xaa, 0x4a, 0x32, 0x78, - 0xc8, 0xaa, 0x6a, 0x6b, 0x10, 0x39, 0x5f, 0x09, 0xec, 0xae, 0x70, 0x11, 0x29, 0x4f, 0x04, 0xd2, - 0xfb, 0x50, 0x15, 0x3a, 0xa3, 0x5d, 0x6a, 0x9e, 0xe3, 0x9e, 0x3f, 0x90, 0x3b, 0x10, 0x62, 0x16, - 0x24, 0xe1, 0x42, 0x5b, 0x2a, 0x68, 0x0f, 0x6a, 0xe1, 0xd2, 0xb8, 0x69, 0x69, 0x83, 0x8e, 0xc9, - 0xe0, 0xc4, 0xfb, 0xd9, 0x49, 0x8d, 0xf3, 0x9e, 0xc0, 0x8e, 0x72, 0xc7, 0x33, 0x94, 0x8b, 0x53, - 0xde, 0x83, 0x4b, 0x19, 0x9f, 0xa0, 0x86, 0xab, 0x7b, 0x6d, 0x93, 0xb7, 0x52, 0x32, 0x3e, 0xc1, - 0xbe, 0xd5, 0x24, 0x4c, 0x57, 0xd3, 0x9b, 0x50, 0x09, 0x45, 0xa6, 0x81, 0xb6, 0xfb, 0x9b, 0xf9, - 0x71, 0xa7, 0xe2, 0x0f, 0x19, 0x53, 0x39, 0xda, 0x80, 0xcb, 0x92, 0x8f, 0x31, 0x69, 0x56, 0x54, - 0xd3, 0x58, 0x11, 0x38, 0x9f, 0x08, 0xb4, 0xcd, 0x18, 0x65, 0x9b, 0x2e, 0xd2, 0x6d, 0xfa, 0x14, - 0xae, 0xea, 0xa2, 0x29, 0x4e, 0x0f, 0x31, 0x13, 0x2f, 0xe2, 0x54, 0x23, 0xd4, 0xbd, 0xdb, 0xab, - 0xb8, 0x87, 0x29, 0x86, 0xee, 0xe3, 0x7f, 0xe5, 0xac, 0xae, 0xf4, 0xcb, 0xd8, 0xd9, 0x85, 0x9d, - 0x47, 0x28, 0x19, 0xe7, 0xd2, 0xef, 0x9d, 0xef, 0x8e, 0xf3, 0x00, 0xda, 0xe6, 0xed, 0x92, 0x7a, - 0xef, 0xf4, 0x80, 0x14, 0xf9, 0xf6, 0xa9, 0xfe, 0x7b, 0x1f, 0x09, 0x58, 0x7e, 0x8f, 0xbe, 0x23, - 0xd0, 0x30, 0x39, 0xd1, 0xae, 0x89, 0x7c, 0x0d, 0x52, 0x6b, 0xff, 0xe2, 0x82, 0x02, 0xd2, 0xd9, - 0xfa, 0xfe, 0xed, 0xf7, 0x17, 0xcb, 0xba, 0x46, 0xbc, 0xcf, 0x16, 0xe8, 0x96, 0x96, 0x40, 0xa6, - 0x81, 0x98, 0x81, 0xd6, 0x7c, 0x41, 0x66, 0xa0, 0x75, 0xb3, 0x5e, 0x02, 0xd1, 0x0f, 0x04, 0x6e, - 0x18, 0xaf, 0x0f, 0xdd, 0x5f, 0x35, 0xd1, 0x55, 0xf7, 0xb5, 0x75, 0xf0, 0x1f, 0x8a, 0xb3, 0x20, - 0xfd, 0xf6, 0xd1, 0xdc, 0xde, 0xf8, 0x39, 0xb7, 0x37, 0xfe, 0xcc, 0x6d, 0xf2, 0x36, 0xb7, 0xc9, - 0x51, 0x6e, 0x93, 0x1f, 0xb9, 0x4d, 0x7e, 0xe5, 0x36, 0x39, 0xac, 0xea, 0x3f, 0xc6, 0xdd, 0xbf, - 0x01, 0x00, 0x00, 0xff, 0xff, 0xb3, 0xf8, 0x41, 0xef, 0x96, 0x04, 0x00, 0x00, + // 586 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcb, 0x6e, 0xd3, 0x40, + 0x14, 0xcd, 0x38, 0x25, 0x69, 0x6f, 0x42, 0x8a, 0xa6, 0x89, 0x14, 0xf2, 0x70, 0x2a, 0xb3, 0x68, + 0x37, 0x38, 0x6d, 0x60, 0x05, 0x1b, 0x92, 0x20, 0x55, 0x11, 0x02, 0x21, 0x47, 0xb0, 0xad, 0x5c, + 0x67, 0x08, 0x56, 0x12, 0x8f, 0xf1, 0x8c, 0x0b, 0xd9, 0x21, 0x51, 0xf1, 0x07, 0x08, 0x56, 0x7c, + 0x02, 0xdf, 0x11, 0xb1, 0x62, 0xc9, 0x2a, 0x22, 0xfe, 0x00, 0xc4, 0x27, 0x20, 0x8f, 0x6d, 0x9a, + 0x87, 0x13, 0xda, 0x55, 0x3c, 0xd7, 0xe7, 0x9c, 0x7b, 0xee, 0xc9, 0xf5, 0xc0, 0xb6, 0xa1, 0xab, + 0xb6, 0x43, 0x39, 0xc5, 0xb8, 0x47, 0x8d, 0x01, 0x71, 0x54, 0xf6, 0x56, 0x77, 0x46, 0x03, 0x93, + 0xab, 0xe7, 0xc7, 0xa5, 0x0c, 0x1f, 0xdb, 0x84, 0x05, 0x80, 0x52, 0x86, 0xd9, 0xc4, 0x88, 0x0e, + 0xf9, 0x3e, 0xed, 0x53, 0xf1, 0x58, 0xf7, 0x9f, 0xc2, 0xea, 0x9e, 0x3d, 0x74, 0xfb, 0xa6, 0x55, + 0x0f, 0x7e, 0x82, 0xa2, 0xd2, 0x86, 0xca, 0x33, 0xda, 0x23, 0x6d, 0xe2, 0x70, 0xf3, 0x95, 0x69, + 0xe8, 0x9c, 0x74, 0xb9, 0xce, 0x5d, 0xa6, 0x91, 0x37, 0x2e, 0x61, 0x1c, 0xdf, 0x81, 0xb4, 0x45, + 0x7b, 0xe4, 0xd4, 0xec, 0x15, 0xd1, 0x3e, 0x3a, 0xdc, 0x69, 0x81, 0x37, 0xad, 0xa5, 0x7c, 0x4a, + 0xe7, 0xb1, 0x96, 0xf2, 0x5f, 0x75, 0x7a, 0xca, 0x57, 0x04, 0xd5, 0x35, 0x2a, 0xcc, 0xa6, 0x16, + 0x23, 0xf8, 0x01, 0xa4, 0x98, 0xa8, 0x08, 0x95, 0x4c, 0x43, 0x51, 0x57, 0x07, 0x52, 0x3b, 0x8c, + 0xb9, 0xba, 0x65, 0x44, 0xdc, 0x90, 0x81, 0x9b, 0x90, 0x31, 0x2e, 0x85, 0x8b, 0x92, 0x10, 0xa8, + 0xc5, 0x09, 0xcc, 0xf5, 0xd7, 0xe6, 0x39, 0xca, 0x05, 0x82, 0xb2, 0xaf, 0x4e, 0x96, 0x5c, 0x46, + 0x53, 0xde, 0x87, 0x2d, 0x87, 0x0e, 0x89, 0x30, 0x97, 0x6b, 0x54, 0xe2, 0xb4, 0x7d, 0xa6, 0x46, + 0x87, 0xa4, 0x25, 0x15, 0x91, 0x26, 0xd0, 0xf8, 0x36, 0x24, 0x0d, 0xe6, 0x08, 0x43, 0xd9, 0x56, + 0xda, 0x9b, 0xd6, 0x92, 0xed, 0xae, 0xa6, 0xf9, 0x35, 0x9c, 0x87, 0x1b, 0x9c, 0x0e, 0x88, 0x55, + 0x4c, 0xfa, 0xa1, 0x69, 0xc1, 0x41, 0xf9, 0x84, 0xa0, 0x12, 0x6f, 0x23, 0x8c, 0xe9, 0x2a, 0x69, + 0xe3, 0xe7, 0xb0, 0x2b, 0x40, 0x23, 0x32, 0x3a, 0x23, 0x0e, 0x7b, 0x6d, 0xda, 0xc2, 0x42, 0xae, + 0x71, 0xb0, 0xce, 0x77, 0xd7, 0x26, 0x86, 0xfa, 0xf4, 0x1f, 0x5c, 0xcb, 0xf9, 0xfc, 0xcb, 0xb3, + 0x52, 0x85, 0xf2, 0x09, 0xe1, 0x1a, 0xa5, 0xbc, 0xdd, 0x5c, 0x4d, 0x47, 0x79, 0x04, 0x95, 0xf8, + 0xd7, 0xa1, 0xeb, 0xfd, 0xc5, 0x3f, 0xc8, 0x77, 0x9e, 0x5d, 0xcc, 0xbf, 0x00, 0x7b, 0x27, 0x84, + 0xbf, 0xb0, 0x86, 0xd4, 0x18, 0x3c, 0x21, 0xe3, 0x48, 0xd8, 0x81, 0xfc, 0x62, 0x39, 0x14, 0xac, + 0x02, 0xb8, 0xa2, 0x78, 0x3a, 0x20, 0xe3, 0x50, 0x6f, 0xc7, 0x8d, 0x60, 0xf8, 0x21, 0xa4, 0xcf, + 0x89, 0xc3, 0x4c, 0x6a, 0x85, 0xcb, 0x50, 0x8e, 0x1b, 0xfc, 0x65, 0x00, 0x69, 0x6d, 0x4d, 0xa6, + 0xb5, 0x84, 0x16, 0x31, 0x1a, 0x17, 0x12, 0x48, 0xed, 0x26, 0xfe, 0x80, 0x44, 0xef, 0x95, 0xa1, + 0x70, 0x3d, 0x4e, 0x6b, 0x43, 0x3a, 0xa5, 0xa3, 0xab, 0x13, 0x82, 0xf1, 0x94, 0xed, 0xef, 0xdf, + 0x7e, 0x7f, 0x91, 0xa4, 0x5b, 0x08, 0xbf, 0x83, 0xec, 0x7c, 0x00, 0xf8, 0x60, 0x8d, 0xd6, 0x72, + 0x72, 0xa5, 0xc3, 0xff, 0x03, 0xc3, 0x66, 0x05, 0xd1, 0x6c, 0x17, 0x6e, 0x0a, 0xe4, 0xdd, 0x91, + 0x6e, 0xe9, 0x7d, 0xe2, 0x34, 0x3e, 0x4b, 0x20, 0xf6, 0x2a, 0x8c, 0x22, 0x6e, 0x2b, 0xe3, 0xa3, + 0xd8, 0xf0, 0x19, 0xc5, 0x47, 0xb1, 0x69, 0xe1, 0xe7, 0xa2, 0xf8, 0x88, 0xa0, 0x10, 0x7b, 0x87, + 0xe0, 0xa3, 0x75, 0x6b, 0xbd, 0xee, 0xd2, 0x2a, 0x1d, 0x5f, 0x83, 0xb1, 0x6c, 0xa4, 0x55, 0x99, + 0xcc, 0xe4, 0xc4, 0xcf, 0x99, 0x9c, 0xf8, 0x33, 0x93, 0xd1, 0x7b, 0x4f, 0x46, 0x13, 0x4f, 0x46, + 0x3f, 0x3c, 0x19, 0xfd, 0xf2, 0x64, 0x74, 0x96, 0x12, 0xd7, 0xe6, 0xbd, 0xbf, 0x01, 0x00, 0x00, + 0xff, 0xff, 0xe7, 0x80, 0x3b, 0x00, 0x9b, 0x05, 0x00, 0x00, } diff --git a/api/ca.proto b/api/ca.proto index 2bfa9f7f81..5aa1f673ee 100644 --- a/api/ca.proto +++ b/api/ca.proto @@ -13,6 +13,11 @@ service CA { rpc GetRootCACertificate(GetRootCACertificateRequest) returns (GetRootCACertificateResponse) { option (docker.protobuf.plugin.tls_authorization) = { insecure: true }; }; + // GetUnlockKey returns the current unlock key for the cluster for the role of the client + // asking. + rpc GetUnlockKey(GetUnlockKeyRequest) returns (GetUnlockKeyResponse) { + option (docker.protobuf.plugin.tls_authorization) = { roles: ["swarm-manager"] }; + }; } service NodeCA { @@ -55,3 +60,10 @@ message GetRootCACertificateRequest {} message GetRootCACertificateResponse { bytes certificate = 1; } + +message GetUnlockKeyRequest {} + +message GetUnlockKeyResponse { + bytes unlock_key = 1; + Version version = 2 [(gogoproto.nullable) = false]; +} diff --git a/api/control.pb.go b/api/control.pb.go index 0cff716d22..6f36208c2c 100644 --- a/api/control.pb.go +++ b/api/control.pb.go @@ -405,16 +405,19 @@ func (m *ListClustersResponse) Reset() { *m = ListClustersRes func (*ListClustersResponse) ProtoMessage() {} func (*ListClustersResponse) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{35} } -type JoinTokenRotation struct { - // RotateWorkerToken tells UpdateCluster to rotate the worker secret. - RotateWorkerToken bool `protobuf:"varint,1,opt,name=rotate_worker_token,json=rotateWorkerToken,proto3" json:"rotate_worker_token,omitempty"` - // RotateManagerSecret tells UpdateCluster to rotate the manager secret. - RotateManagerToken bool `protobuf:"varint,2,opt,name=rotate_manager_token,json=rotateManagerToken,proto3" json:"rotate_manager_token,omitempty"` +// KeyRotation tells UpdateCluster what items to rotate +type KeyRotation struct { + // WorkerJoinToken tells UpdateCluster to rotate the worker secret token. + WorkerJoinToken bool `protobuf:"varint,1,opt,name=worker_join_token,json=workerJoinToken,proto3" json:"worker_join_token,omitempty"` + // ManagerJoinToken tells UpdateCluster to rotate the manager secret token. + ManagerJoinToken bool `protobuf:"varint,2,opt,name=manager_join_token,json=managerJoinToken,proto3" json:"manager_join_token,omitempty"` + // ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key + ManagerUnlockKey bool `protobuf:"varint,3,opt,name=manager_unlock_key,json=managerUnlockKey,proto3" json:"manager_unlock_key,omitempty"` } -func (m *JoinTokenRotation) Reset() { *m = JoinTokenRotation{} } -func (*JoinTokenRotation) ProtoMessage() {} -func (*JoinTokenRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} } +func (m *KeyRotation) Reset() { *m = KeyRotation{} } +func (*KeyRotation) ProtoMessage() {} +func (*KeyRotation) Descriptor() ([]byte, []int) { return fileDescriptorControl, []int{36} } type UpdateClusterRequest struct { // ClusterID is the cluster ID to update. @@ -423,8 +426,8 @@ type UpdateClusterRequest struct { ClusterVersion *Version `protobuf:"bytes,2,opt,name=cluster_version,json=clusterVersion" json:"cluster_version,omitempty"` // Spec is the new spec to apply to the cluster. Spec *ClusterSpec `protobuf:"bytes,3,opt,name=spec" json:"spec,omitempty"` - // Rotation contains flags for join token rotation - Rotation JoinTokenRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"` + // Rotation contains flags for join token and unlock key rotation + Rotation KeyRotation `protobuf:"bytes,4,opt,name=rotation" json:"rotation"` } func (m *UpdateClusterRequest) Reset() { *m = UpdateClusterRequest{} } @@ -598,7 +601,7 @@ func init() { proto.RegisterType((*ListClustersRequest)(nil), "docker.swarmkit.v1.ListClustersRequest") proto.RegisterType((*ListClustersRequest_Filters)(nil), "docker.swarmkit.v1.ListClustersRequest.Filters") proto.RegisterType((*ListClustersResponse)(nil), "docker.swarmkit.v1.ListClustersResponse") - proto.RegisterType((*JoinTokenRotation)(nil), "docker.swarmkit.v1.JoinTokenRotation") + proto.RegisterType((*KeyRotation)(nil), "docker.swarmkit.v1.KeyRotation") proto.RegisterType((*UpdateClusterRequest)(nil), "docker.swarmkit.v1.UpdateClusterRequest") proto.RegisterType((*UpdateClusterResponse)(nil), "docker.swarmkit.v1.UpdateClusterResponse") proto.RegisterType((*GetSecretRequest)(nil), "docker.swarmkit.v1.GetSecretRequest") @@ -1459,14 +1462,15 @@ func (m *ListClustersResponse) Copy() *ListClustersResponse { return o } -func (m *JoinTokenRotation) Copy() *JoinTokenRotation { +func (m *KeyRotation) Copy() *KeyRotation { if m == nil { return nil } - o := &JoinTokenRotation{ - RotateWorkerToken: m.RotateWorkerToken, - RotateManagerToken: m.RotateManagerToken, + o := &KeyRotation{ + WorkerJoinToken: m.WorkerJoinToken, + ManagerJoinToken: m.ManagerJoinToken, + ManagerUnlockKey: m.ManagerUnlockKey, } return o @@ -2199,14 +2203,15 @@ func (this *ListClustersResponse) GoString() string { s = append(s, "}") return strings.Join(s, "") } -func (this *JoinTokenRotation) GoString() string { +func (this *KeyRotation) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 6) - s = append(s, "&api.JoinTokenRotation{") - s = append(s, "RotateWorkerToken: "+fmt.Sprintf("%#v", this.RotateWorkerToken)+",\n") - s = append(s, "RotateManagerToken: "+fmt.Sprintf("%#v", this.RotateManagerToken)+",\n") + s := make([]string, 0, 7) + s = append(s, "&api.KeyRotation{") + s = append(s, "WorkerJoinToken: "+fmt.Sprintf("%#v", this.WorkerJoinToken)+",\n") + s = append(s, "ManagerJoinToken: "+fmt.Sprintf("%#v", this.ManagerJoinToken)+",\n") + s = append(s, "ManagerUnlockKey: "+fmt.Sprintf("%#v", this.ManagerUnlockKey)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -4734,7 +4739,7 @@ func (m *ListClustersResponse) MarshalTo(data []byte) (int, error) { return i, nil } -func (m *JoinTokenRotation) Marshal() (data []byte, err error) { +func (m *KeyRotation) Marshal() (data []byte, err error) { size := m.Size() data = make([]byte, size) n, err := m.MarshalTo(data) @@ -4744,25 +4749,35 @@ func (m *JoinTokenRotation) Marshal() (data []byte, err error) { return data[:n], nil } -func (m *JoinTokenRotation) MarshalTo(data []byte) (int, error) { +func (m *KeyRotation) MarshalTo(data []byte) (int, error) { var i int _ = i var l int _ = l - if m.RotateWorkerToken { + if m.WorkerJoinToken { data[i] = 0x8 i++ - if m.RotateWorkerToken { + if m.WorkerJoinToken { data[i] = 1 } else { data[i] = 0 } i++ } - if m.RotateManagerToken { + if m.ManagerJoinToken { data[i] = 0x10 i++ - if m.RotateManagerToken { + if m.ManagerJoinToken { + data[i] = 1 + } else { + data[i] = 0 + } + i++ + } + if m.ManagerUnlockKey { + data[i] = 0x18 + i++ + if m.ManagerUnlockKey { data[i] = 1 } else { data[i] = 0 @@ -6618,13 +6633,16 @@ func (m *ListClustersResponse) Size() (n int) { return n } -func (m *JoinTokenRotation) Size() (n int) { +func (m *KeyRotation) Size() (n int) { var l int _ = l - if m.RotateWorkerToken { + if m.WorkerJoinToken { + n += 2 + } + if m.ManagerJoinToken { n += 2 } - if m.RotateManagerToken { + if m.ManagerUnlockKey { n += 2 } return n @@ -7294,13 +7312,14 @@ func (this *ListClustersResponse) String() string { }, "") return s } -func (this *JoinTokenRotation) String() string { +func (this *KeyRotation) String() string { if this == nil { return "nil" } - s := strings.Join([]string{`&JoinTokenRotation{`, - `RotateWorkerToken:` + fmt.Sprintf("%v", this.RotateWorkerToken) + `,`, - `RotateManagerToken:` + fmt.Sprintf("%v", this.RotateManagerToken) + `,`, + s := strings.Join([]string{`&KeyRotation{`, + `WorkerJoinToken:` + fmt.Sprintf("%v", this.WorkerJoinToken) + `,`, + `ManagerJoinToken:` + fmt.Sprintf("%v", this.ManagerJoinToken) + `,`, + `ManagerUnlockKey:` + fmt.Sprintf("%v", this.ManagerUnlockKey) + `,`, `}`, }, "") return s @@ -7313,7 +7332,7 @@ func (this *UpdateClusterRequest) String() string { `ClusterID:` + fmt.Sprintf("%v", this.ClusterID) + `,`, `ClusterVersion:` + strings.Replace(fmt.Sprintf("%v", this.ClusterVersion), "Version", "Version", 1) + `,`, `Spec:` + strings.Replace(fmt.Sprintf("%v", this.Spec), "ClusterSpec", "ClusterSpec", 1) + `,`, - `Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "JoinTokenRotation", "JoinTokenRotation", 1), `&`, ``, 1) + `,`, + `Rotation:` + strings.Replace(strings.Replace(this.Rotation.String(), "KeyRotation", "KeyRotation", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -11855,7 +11874,7 @@ func (m *ListClustersResponse) Unmarshal(data []byte) error { } return nil } -func (m *JoinTokenRotation) Unmarshal(data []byte) error { +func (m *KeyRotation) Unmarshal(data []byte) error { l := len(data) iNdEx := 0 for iNdEx < l { @@ -11878,15 +11897,15 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: JoinTokenRotation: wiretype end group for non-group") + return fmt.Errorf("proto: KeyRotation: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: JoinTokenRotation: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: KeyRotation: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RotateWorkerToken", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field WorkerJoinToken", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -11903,10 +11922,30 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error { break } } - m.RotateWorkerToken = bool(v != 0) + m.WorkerJoinToken = bool(v != 0) case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RotateManagerToken", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ManagerJoinToken", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowControl + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ManagerJoinToken = bool(v != 0) + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ManagerUnlockKey", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -11923,7 +11962,7 @@ func (m *JoinTokenRotation) Unmarshal(data []byte) error { break } } - m.RotateManagerToken = bool(v != 0) + m.ManagerUnlockKey = bool(v != 0) default: iNdEx = preIndex skippy, err := skipControl(data[iNdEx:]) @@ -13413,117 +13452,117 @@ var ( func init() { proto.RegisterFile("control.proto", fileDescriptorControl) } var fileDescriptorControl = []byte{ - // 1777 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcf, 0x6f, 0xdb, 0xc6, - 0x12, 0x8e, 0x24, 0xdb, 0xb2, 0x47, 0x96, 0x13, 0xaf, 0x95, 0x3c, 0x81, 0xc9, 0x93, 0x03, 0xe6, - 0xc5, 0x91, 0x81, 0x3c, 0x39, 0x4f, 0x79, 0x41, 0xd3, 0x14, 0xfd, 0x65, 0xbb, 0x71, 0x95, 0x1f, - 0x6e, 0x40, 0x27, 0x6d, 0x6f, 0x86, 0x2c, 0x6d, 0x5c, 0x56, 0xb2, 0xa8, 0x92, 0xb4, 0x93, 0xa0, - 0x97, 0x16, 0x68, 0x81, 0xfe, 0x09, 0xbd, 0xf6, 0xda, 0x02, 0x3d, 0xf7, 0xd6, 0x6b, 0xd0, 0x53, - 0x8f, 0x3d, 0x19, 0x8d, 0x80, 0x02, 0x3d, 0x15, 0xfd, 0x0b, 0x8a, 0x62, 0x77, 0x67, 0x49, 0x8a, - 0x5a, 0x92, 0x92, 0xe5, 0xc2, 0x39, 0x99, 0x5c, 0x7e, 0xb3, 0x33, 0xbb, 0xf3, 0xed, 0xa7, 0xd9, - 0x81, 0x21, 0xdf, 0xb0, 0x3a, 0xae, 0x6d, 0xb5, 0x2b, 0x5d, 0xdb, 0x72, 0x2d, 0x42, 0x9a, 0x56, - 0xa3, 0x45, 0xed, 0x8a, 0xf3, 0xa4, 0x6e, 0xef, 0xb5, 0x4c, 0xb7, 0x72, 0xf0, 0x3f, 0x2d, 0xe7, - 0x74, 0x69, 0xc3, 0x11, 0x00, 0x2d, 0x6f, 0xed, 0x7c, 0x4c, 0x1b, 0xae, 0x7c, 0xcd, 0xb9, 0xcf, - 0xba, 0x54, 0xbe, 0x14, 0x76, 0xad, 0x5d, 0x8b, 0x3f, 0xae, 0xb0, 0x27, 0x1c, 0x5d, 0xe8, 0xb6, - 0xf7, 0x77, 0xcd, 0xce, 0x8a, 0xf8, 0x23, 0x06, 0xf5, 0x1b, 0x30, 0xb7, 0x41, 0xdd, 0x4d, 0xab, - 0x49, 0x0d, 0xfa, 0xc9, 0x3e, 0x75, 0x5c, 0x72, 0x09, 0xb2, 0x1d, 0xab, 0x49, 0xb7, 0xcd, 0x66, - 0x31, 0x75, 0x31, 0x55, 0x9e, 0x59, 0x85, 0xde, 0xe1, 0xe2, 0x14, 0x43, 0xd4, 0xd6, 0x8d, 0x29, - 0xf6, 0xa9, 0xd6, 0xd4, 0xdf, 0x84, 0xd3, 0x9e, 0x99, 0xd3, 0xb5, 0x3a, 0x0e, 0x25, 0x57, 0x61, - 0x82, 0x7d, 0xe4, 0x46, 0xb9, 0x6a, 0xb1, 0x32, 0xb8, 0x80, 0x0a, 0xc7, 0x73, 0x94, 0x7e, 0x98, - 0x81, 0x33, 0xf7, 0x4c, 0x87, 0x4f, 0xe1, 0x48, 0xd7, 0xb7, 0x21, 0xfb, 0xd8, 0x6c, 0xbb, 0xd4, - 0x76, 0x70, 0x96, 0xab, 0xaa, 0x59, 0xc2, 0x66, 0x95, 0xdb, 0xc2, 0xc6, 0x90, 0xc6, 0xda, 0xe7, - 0x19, 0xc8, 0xe2, 0x20, 0x29, 0xc0, 0x64, 0xa7, 0xbe, 0x47, 0xd9, 0x8c, 0x99, 0xf2, 0x8c, 0x21, - 0x5e, 0xc8, 0x0a, 0xe4, 0xcc, 0xe6, 0x76, 0xd7, 0xa6, 0x8f, 0xcd, 0xa7, 0xd4, 0x29, 0xa6, 0xd9, - 0xb7, 0xd5, 0xb9, 0xde, 0xe1, 0x22, 0xd4, 0xd6, 0x1f, 0xe0, 0xa8, 0x01, 0x66, 0x53, 0x3e, 0x93, - 0x07, 0x30, 0xd5, 0xae, 0xef, 0xd0, 0xb6, 0x53, 0xcc, 0x5c, 0xcc, 0x94, 0x73, 0xd5, 0x9b, 0xa3, - 0x44, 0x56, 0xb9, 0xc7, 0x4d, 0xdf, 0xe9, 0xb8, 0xf6, 0x33, 0x03, 0xe7, 0x21, 0x35, 0xc8, 0xed, - 0xd1, 0xbd, 0x1d, 0x6a, 0x3b, 0x1f, 0x99, 0x5d, 0xa7, 0x38, 0x71, 0x31, 0x53, 0x9e, 0xab, 0x5e, - 0x89, 0xda, 0xb6, 0xad, 0x2e, 0x6d, 0x54, 0xee, 0x7b, 0x78, 0x23, 0x68, 0x4b, 0xaa, 0x30, 0x69, - 0x5b, 0x6d, 0xea, 0x14, 0x27, 0xf9, 0x24, 0x17, 0x22, 0xf7, 0xde, 0x6a, 0x53, 0x43, 0x40, 0xc9, - 0x25, 0xc8, 0xb3, 0xad, 0xf0, 0xf7, 0x60, 0x8a, 0xef, 0xcf, 0x2c, 0x1b, 0x94, 0xab, 0xd6, 0x5e, - 0x85, 0x5c, 0x20, 0x74, 0x72, 0x06, 0x32, 0x2d, 0xfa, 0x4c, 0xd0, 0xc2, 0x60, 0x8f, 0x6c, 0x77, - 0x0f, 0xea, 0xed, 0x7d, 0x5a, 0x4c, 0xf3, 0x31, 0xf1, 0x72, 0x2b, 0x7d, 0x33, 0xa5, 0xaf, 0xc1, - 0x7c, 0x60, 0x3b, 0x90, 0x23, 0x15, 0x98, 0x64, 0xd9, 0x17, 0xc9, 0x88, 0x23, 0x89, 0x80, 0xe9, - 0xdf, 0xa6, 0x60, 0xfe, 0x51, 0xb7, 0x59, 0x77, 0xe9, 0xa8, 0x0c, 0x25, 0x6f, 0xc0, 0x2c, 0x07, - 0x1d, 0x50, 0xdb, 0x31, 0xad, 0x0e, 0x0f, 0x30, 0x57, 0x3d, 0xaf, 0xf2, 0xf8, 0xbe, 0x80, 0x18, - 0x39, 0x66, 0x80, 0x2f, 0xe4, 0x1a, 0x4c, 0xb0, 0xe3, 0x56, 0xcc, 0x70, 0xbb, 0x0b, 0x71, 0x79, - 0x31, 0x38, 0x52, 0x5f, 0x05, 0x12, 0x8c, 0xf5, 0x48, 0xc7, 0x62, 0x13, 0xe6, 0x0d, 0xba, 0x67, - 0x1d, 0x8c, 0xbe, 0xde, 0x02, 0x4c, 0x3e, 0xb6, 0xec, 0x86, 0xc8, 0xc4, 0xb4, 0x21, 0x5e, 0xf4, - 0x02, 0x90, 0xe0, 0x7c, 0x22, 0x26, 0x3c, 0xf4, 0x0f, 0xeb, 0x4e, 0x2b, 0xe0, 0xc2, 0xad, 0x3b, - 0xad, 0x90, 0x0b, 0x86, 0x60, 0x2e, 0xd8, 0x27, 0xef, 0xd0, 0x0b, 0x33, 0x7f, 0x75, 0xec, 0x63, - 0xdc, 0xea, 0x38, 0x9e, 0xa3, 0xf4, 0x9b, 0x72, 0x75, 0x23, 0xbb, 0xf6, 0xd6, 0x11, 0xf4, 0xae, - 0xff, 0x85, 0x22, 0xc2, 0x06, 0x8f, 0x20, 0x22, 0x41, 0xb3, 0x41, 0x11, 0xf9, 0xe6, 0x04, 0x45, - 0x44, 0x15, 0x99, 0x52, 0x44, 0x56, 0x20, 0xe7, 0x50, 0xfb, 0xc0, 0x6c, 0x30, 0x76, 0x08, 0x11, - 0xc1, 0x10, 0xb6, 0xc4, 0x70, 0x6d, 0xdd, 0x31, 0x00, 0x21, 0xb5, 0xa6, 0x43, 0x96, 0x60, 0x1a, - 0xb9, 0x24, 0xd4, 0x62, 0x66, 0x35, 0xd7, 0x3b, 0x5c, 0xcc, 0x0a, 0x32, 0x39, 0x46, 0x56, 0xb0, - 0xc9, 0x21, 0xeb, 0x30, 0xd7, 0xa4, 0x8e, 0x69, 0xd3, 0xe6, 0xb6, 0xe3, 0xd6, 0x5d, 0xd4, 0x87, - 0xb9, 0xea, 0xbf, 0xa3, 0x52, 0xbc, 0xc5, 0x50, 0x46, 0x1e, 0x8d, 0xf8, 0x9b, 0x42, 0x64, 0xb2, - 0xff, 0x88, 0xc8, 0xe0, 0x76, 0xf9, 0x22, 0xc3, 0x58, 0x13, 0x2b, 0x32, 0x9c, 0x46, 0x02, 0xa6, - 0xdf, 0x85, 0xc2, 0x9a, 0x4d, 0xeb, 0x2e, 0xc5, 0x2d, 0x93, 0x44, 0xba, 0x8e, 0x0a, 0x20, 0x58, - 0xb4, 0xa8, 0x9a, 0x06, 0x2d, 0x02, 0x22, 0xb0, 0x09, 0x67, 0x43, 0x93, 0x61, 0x54, 0x37, 0x20, - 0x8b, 0x69, 0xc0, 0x09, 0xcf, 0xc7, 0x4c, 0x68, 0x48, 0xac, 0xfe, 0x36, 0xcc, 0x6f, 0x50, 0x37, - 0x14, 0xd9, 0x55, 0x00, 0x3f, 0xeb, 0x78, 0x6a, 0xf2, 0xbd, 0xc3, 0xc5, 0x19, 0x2f, 0xe9, 0xc6, - 0x8c, 0x97, 0x73, 0xfd, 0x2e, 0x90, 0xe0, 0x14, 0xe3, 0xc5, 0xf3, 0x63, 0x0a, 0x0a, 0x42, 0xe5, - 0xc6, 0x89, 0x89, 0xac, 0xc3, 0x69, 0x89, 0x1e, 0x41, 0xa0, 0xe7, 0xd0, 0x46, 0x6a, 0xf4, 0xf5, - 0x3e, 0x8d, 0x1e, 0x3e, 0x43, 0xa1, 0x05, 0x8c, 0xb7, 0x23, 0xeb, 0x50, 0x10, 0xd2, 0x34, 0x56, - 0x92, 0xfe, 0x05, 0x67, 0x43, 0xb3, 0xa0, 0xc6, 0xfd, 0x9e, 0x86, 0x05, 0xc6, 0x71, 0x1c, 0xf7, - 0x64, 0xae, 0x16, 0x96, 0xb9, 0x95, 0x28, 0x31, 0x09, 0x59, 0x0e, 0x2a, 0xdd, 0x97, 0xe9, 0x63, - 0x57, 0xba, 0xad, 0x90, 0xd2, 0xbd, 0x36, 0x62, 0x70, 0x4a, 0xb1, 0x1b, 0x50, 0x93, 0x89, 0xe3, - 0x55, 0x93, 0xf7, 0xa0, 0xd0, 0x1f, 0x12, 0x12, 0xe3, 0x15, 0x98, 0xc6, 0x44, 0x49, 0x4d, 0x89, - 0x65, 0x86, 0x07, 0xf6, 0x95, 0x65, 0x93, 0xba, 0x4f, 0x2c, 0xbb, 0x35, 0x82, 0xb2, 0xa0, 0x85, - 0x4a, 0x59, 0xbc, 0xc9, 0x7c, 0xde, 0x76, 0xc4, 0x50, 0x1c, 0x6f, 0xa5, 0x95, 0xc4, 0xea, 0x8f, - 0xb8, 0xb2, 0x84, 0x22, 0x23, 0x30, 0xc1, 0x76, 0x13, 0xf7, 0x8b, 0x3f, 0x33, 0x22, 0xa3, 0x0d, - 0x23, 0x72, 0xda, 0x27, 0x32, 0xda, 0x32, 0x22, 0x23, 0xc0, 0x53, 0x9b, 0x63, 0x8a, 0xf1, 0x43, - 0x79, 0xb6, 0x8e, 0x3d, 0x4c, 0xef, 0xbc, 0x85, 0x22, 0xf5, 0xce, 0x1b, 0x8e, 0x1f, 0xe1, 0xbc, - 0x85, 0x2c, 0x5f, 0xae, 0xf3, 0x16, 0x11, 0xdc, 0x49, 0x9e, 0x37, 0x3f, 0x24, 0xff, 0xbc, 0x61, - 0xa2, 0x62, 0xcf, 0x9b, 0xcc, 0x9c, 0x07, 0xc6, 0x1f, 0xcb, 0xb5, 0xf6, 0xbe, 0xe3, 0x52, 0x3b, - 0xa0, 0xc3, 0x0d, 0x31, 0x12, 0xd2, 0x61, 0xc4, 0x31, 0x5e, 0x20, 0xc0, 0xa3, 0xaf, 0x37, 0x85, - 0x4f, 0x5f, 0x84, 0xc4, 0xd1, 0x57, 0x5a, 0x49, 0xac, 0xc7, 0x25, 0xfc, 0x70, 0x04, 0x2e, 0x85, - 0x2c, 0x5f, 0x2e, 0x2e, 0x45, 0x04, 0x77, 0x92, 0x5c, 0xf2, 0x43, 0xf2, 0xb9, 0x84, 0xd9, 0x88, - 0xe5, 0x92, 0x4c, 0x9d, 0x07, 0xd6, 0xf7, 0x61, 0xfe, 0x8e, 0x65, 0x76, 0x1e, 0x5a, 0x2d, 0xda, - 0x31, 0x2c, 0xb7, 0xee, 0xb2, 0x82, 0xa3, 0x02, 0x0b, 0x36, 0x7b, 0xa6, 0xdb, 0x8c, 0x70, 0xd4, - 0xde, 0x76, 0xd9, 0x67, 0x1e, 0xe1, 0xb4, 0x31, 0x2f, 0x3e, 0x7d, 0xc0, 0xbf, 0x70, 0x3b, 0x72, - 0x0d, 0x0a, 0x88, 0xdf, 0xab, 0x77, 0xea, 0xbb, 0x9e, 0x81, 0xb8, 0xa3, 0x11, 0xf1, 0xed, 0xbe, - 0xf8, 0xc4, 0x2d, 0xf4, 0xaf, 0xd2, 0xb2, 0xbe, 0x1a, 0x87, 0xc6, 0xac, 0xbe, 0x92, 0xe8, 0x51, - 0xea, 0x2b, 0xb4, 0x19, 0xa1, 0xbe, 0x42, 0xef, 0xfe, 0xef, 0x14, 0xd9, 0x80, 0x69, 0x1b, 0xf7, - 0xab, 0x38, 0xc1, 0x0d, 0x2f, 0xab, 0x0c, 0x07, 0x36, 0x77, 0x75, 0xe2, 0xf9, 0xe1, 0xe2, 0x29, - 0xc3, 0x33, 0xf6, 0x0b, 0xb5, 0x63, 0x3a, 0x8d, 0xaf, 0xc3, 0x19, 0x5e, 0x07, 0x37, 0x6c, 0xea, - 0xca, 0x5d, 0x5d, 0x86, 0x19, 0x87, 0x0f, 0xf8, 0x9b, 0x3a, 0xdb, 0x3b, 0x5c, 0x9c, 0x16, 0xa8, - 0xda, 0x3a, 0xfb, 0x31, 0xe7, 0x4f, 0x4d, 0x7d, 0x03, 0x2b, 0x71, 0x61, 0x8e, 0xa1, 0x54, 0x61, - 0x4a, 0x00, 0x30, 0x12, 0x4d, 0x5d, 0x18, 0x70, 0x1b, 0x44, 0xea, 0x3f, 0xa4, 0x60, 0x41, 0x56, - 0xa0, 0x47, 0x8b, 0x85, 0xac, 0xc2, 0x1c, 0x42, 0x47, 0xc8, 0x6e, 0x5e, 0x98, 0xc8, 0xe4, 0x56, - 0xfb, 0x92, 0x5b, 0x8a, 0x0e, 0x3c, 0x50, 0x83, 0xdc, 0xf1, 0x8b, 0xff, 0xb1, 0xb7, 0xe1, 0xb7, - 0x34, 0x10, 0x51, 0x6e, 0xb1, 0x57, 0x4f, 0x1b, 0xdf, 0x0d, 0x6b, 0x63, 0x25, 0xba, 0x74, 0x0c, - 0x1a, 0x0e, 0x4a, 0xe3, 0x17, 0xc7, 0x2f, 0x8d, 0x46, 0x48, 0x1a, 0x6f, 0x8d, 0x16, 0xdb, 0x89, - 0x28, 0xe3, 0x5d, 0x79, 0x7f, 0xc0, 0x88, 0x30, 0x65, 0xff, 0x67, 0xb7, 0x1d, 0x3e, 0x84, 0xba, - 0x18, 0x97, 0x33, 0x09, 0xd5, 0x6b, 0xb0, 0x20, 0xaf, 0xb7, 0x41, 0xea, 0x56, 0xfb, 0x0a, 0xda, - 0xa1, 0xb9, 0xd4, 0x3f, 0xd5, 0x18, 0x5c, 0x7a, 0x0b, 0x16, 0xe4, 0xed, 0xe9, 0x88, 0xa7, 0xfb, - 0x9c, 0x7f, 0x8b, 0x0b, 0x46, 0x53, 0xfd, 0xee, 0x1c, 0x64, 0xd7, 0x44, 0x67, 0x9e, 0x98, 0x90, - 0xc5, 0xa6, 0x37, 0xd1, 0x55, 0x41, 0xf5, 0x37, 0xd2, 0xb5, 0x4b, 0xb1, 0x18, 0x2c, 0x37, 0xcf, - 0xfe, 0xf4, 0xfd, 0x1f, 0x5f, 0xa7, 0x4f, 0x43, 0x9e, 0x83, 0xfe, 0x8b, 0x3f, 0x13, 0xc4, 0x82, - 0x19, 0xaf, 0x7b, 0x4a, 0xfe, 0x33, 0x4c, 0xaf, 0x59, 0xbb, 0x9c, 0x80, 0x8a, 0x77, 0x68, 0x03, - 0xf8, 0xcd, 0x4b, 0xa2, 0x9c, 0x6b, 0xa0, 0x11, 0xab, 0x2d, 0x25, 0xc1, 0x12, 0x7d, 0xfa, 0xcd, - 0x49, 0xb5, 0xcf, 0x81, 0x66, 0xa8, 0xda, 0xa7, 0xa2, 0xc7, 0x19, 0xe1, 0x53, 0xe4, 0xf0, 0x61, - 0xdd, 0x69, 0x45, 0xe6, 0x30, 0xd0, 0x9c, 0x8c, 0xcc, 0x61, 0x5f, 0x1b, 0x32, 0x3e, 0x87, 0xbc, - 0x39, 0x15, 0x9d, 0xc3, 0x60, 0xab, 0x2f, 0x3a, 0x87, 0x7d, 0x1d, 0xae, 0xc4, 0xfd, 0xe4, 0xcb, - 0x8b, 0xd9, 0xcf, 0xe0, 0x0a, 0x97, 0x92, 0x60, 0x89, 0x3e, 0xfd, 0xe6, 0x92, 0xda, 0xe7, 0x40, - 0xff, 0x4a, 0xed, 0x73, 0xb0, 0x47, 0x15, 0xe5, 0xf3, 0x29, 0xcc, 0x06, 0xef, 0xe9, 0xe4, 0xca, - 0x90, 0xcd, 0x05, 0xad, 0x9c, 0x0c, 0x8c, 0xf7, 0xfc, 0x29, 0xe4, 0xfb, 0xba, 0x7b, 0x44, 0x39, - 0xa3, 0xaa, 0x9b, 0xa8, 0x2d, 0x0f, 0x81, 0x4c, 0x74, 0xde, 0xd7, 0xb8, 0x52, 0x3b, 0x57, 0x35, - 0xe7, 0xd4, 0xce, 0x95, 0x5d, 0xb0, 0x18, 0xe7, 0x7d, 0xfd, 0x29, 0xb5, 0x73, 0x55, 0x23, 0x4c, - 0xed, 0x5c, 0xdd, 0xec, 0x8a, 0x25, 0x19, 0xde, 0xf7, 0x22, 0x49, 0xd6, 0xdf, 0x23, 0x88, 0x24, - 0x59, 0xf8, 0xc2, 0x1f, 0x4f, 0x32, 0x79, 0x39, 0x8d, 0x26, 0x59, 0xe8, 0x46, 0x1d, 0x4d, 0xb2, - 0xf0, 0x3d, 0x37, 0x91, 0x64, 0x72, 0xc1, 0x31, 0x24, 0x0b, 0xad, 0x79, 0x79, 0x08, 0xe4, 0x90, - 0x79, 0x8e, 0x75, 0xae, 0x6a, 0xca, 0xc4, 0xe5, 0x79, 0x48, 0xe7, 0x22, 0xcf, 0x58, 0xb8, 0x47, - 0xe6, 0xb9, 0xff, 0x62, 0x14, 0x99, 0xe7, 0xd0, 0xad, 0x21, 0x21, 0xcf, 0xf2, 0xe2, 0x18, 0x9d, - 0xe7, 0xd0, 0x6d, 0x37, 0x3a, 0xcf, 0xe1, 0x3b, 0x68, 0xe2, 0x79, 0x96, 0x0b, 0x8e, 0x39, 0xcf, - 0xa1, 0x35, 0x2f, 0x0f, 0x81, 0x4c, 0xfc, 0x71, 0xf2, 0x6e, 0x33, 0xea, 0x1f, 0xa7, 0xf0, 0x5d, - 0x49, 0xbb, 0x9c, 0x80, 0x4a, 0xdc, 0xe7, 0xe0, 0xd5, 0x41, 0xbd, 0xcf, 0x8a, 0x6b, 0x91, 0x56, - 0x4e, 0x06, 0xc6, 0x7b, 0xde, 0x87, 0x5c, 0xa0, 0x00, 0x26, 0x4b, 0xc3, 0xd5, 0xec, 0xda, 0x95, - 0x44, 0x5c, 0xe2, 0x82, 0x83, 0xf5, 0xad, 0x7a, 0xc1, 0x8a, 0x62, 0x5a, 0x2b, 0x27, 0x03, 0x13, - 0x3d, 0x07, 0x6b, 0x59, 0xb5, 0x67, 0x45, 0xbd, 0xac, 0x95, 0x93, 0x81, 0xb1, 0x9e, 0x57, 0x2f, - 0x3c, 0x7f, 0x51, 0x3a, 0xf5, 0xcb, 0x8b, 0xd2, 0xa9, 0x3f, 0x5f, 0x94, 0x52, 0x9f, 0xf5, 0x4a, - 0xa9, 0xe7, 0xbd, 0x52, 0xea, 0xe7, 0x5e, 0x29, 0xf5, 0x6b, 0xaf, 0x94, 0xda, 0x99, 0xe2, 0xff, - 0x72, 0x72, 0xfd, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, 0x2a, 0x7c, 0x4c, 0x3e, 0xeb, 0x22, 0x00, - 0x00, + // 1781 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xcc, 0x5a, 0xcd, 0x6f, 0x1b, 0x45, + 0x14, 0xaf, 0xed, 0x24, 0x4e, 0x9e, 0xe3, 0x7c, 0x4c, 0xdc, 0x62, 0x6d, 0x8b, 0x53, 0x6d, 0x69, + 0xea, 0xa0, 0xe0, 0x80, 0x4b, 0x45, 0x29, 0xe2, 0xa3, 0x8e, 0x69, 0x71, 0x53, 0x42, 0xb5, 0x69, + 0x11, 0xb7, 0xc8, 0xb1, 0xa7, 0x61, 0x6b, 0xc7, 0x6b, 0x76, 0x37, 0x69, 0x23, 0x2e, 0x80, 0xe0, + 0x4f, 0x40, 0xe2, 0xca, 0x15, 0x24, 0xce, 0xdc, 0xb8, 0x56, 0x9c, 0x38, 0x72, 0xb2, 0xa8, 0x25, + 0x24, 0x4e, 0x88, 0xbf, 0x00, 0xa1, 0xf9, 0xda, 0x2f, 0xcf, 0xee, 0xda, 0x71, 0x50, 0x7a, 0x8a, + 0x77, 0xf6, 0xf7, 0xe6, 0xbd, 0x99, 0xf7, 0x9b, 0xdf, 0xbe, 0x79, 0x0a, 0x64, 0x1b, 0x46, 0xc7, + 0x36, 0x8d, 0x76, 0xa9, 0x6b, 0x1a, 0xb6, 0x81, 0x50, 0xd3, 0x68, 0xb4, 0xb0, 0x59, 0xb2, 0x1e, + 0xd7, 0xcd, 0xfd, 0x96, 0x6e, 0x97, 0x0e, 0x5f, 0x53, 0x32, 0x56, 0x17, 0x37, 0x2c, 0x06, 0x50, + 0xb2, 0xc6, 0xee, 0x23, 0xdc, 0xb0, 0xc5, 0x63, 0xc6, 0x3e, 0xea, 0x62, 0xf1, 0x90, 0xdb, 0x33, + 0xf6, 0x0c, 0xfa, 0x73, 0x9d, 0xfc, 0xe2, 0xa3, 0x4b, 0xdd, 0xf6, 0xc1, 0x9e, 0xde, 0x59, 0x67, + 0x7f, 0xd8, 0xa0, 0x7a, 0x0d, 0xe6, 0x6e, 0x63, 0x7b, 0xcb, 0x68, 0x62, 0x0d, 0x7f, 0x76, 0x80, + 0x2d, 0x1b, 0x5d, 0x82, 0x74, 0xc7, 0x68, 0xe2, 0x1d, 0xbd, 0x99, 0x4f, 0x5c, 0x4c, 0x14, 0x67, + 0x2a, 0xd0, 0xef, 0x2d, 0x4f, 0x11, 0x44, 0xad, 0xaa, 0x4d, 0x91, 0x57, 0xb5, 0xa6, 0xfa, 0x2e, + 0xcc, 0x3b, 0x66, 0x56, 0xd7, 0xe8, 0x58, 0x18, 0xad, 0xc1, 0x04, 0x79, 0x49, 0x8d, 0x32, 0xe5, + 0x7c, 0x69, 0x70, 0x01, 0x25, 0x8a, 0xa7, 0x28, 0xb5, 0x97, 0x82, 0x85, 0xbb, 0xba, 0x45, 0xa7, + 0xb0, 0x84, 0xeb, 0x5b, 0x90, 0x7e, 0xa8, 0xb7, 0x6d, 0x6c, 0x5a, 0x7c, 0x96, 0x35, 0xd9, 0x2c, + 0x41, 0xb3, 0xd2, 0x2d, 0x66, 0xa3, 0x09, 0x63, 0xe5, 0xcb, 0x14, 0xa4, 0xf9, 0x20, 0xca, 0xc1, + 0x64, 0xa7, 0xbe, 0x8f, 0xc9, 0x8c, 0xa9, 0xe2, 0x8c, 0xc6, 0x1e, 0xd0, 0x3a, 0x64, 0xf4, 0xe6, + 0x4e, 0xd7, 0xc4, 0x0f, 0xf5, 0x27, 0xd8, 0xca, 0x27, 0xc9, 0xbb, 0xca, 0x5c, 0xbf, 0xb7, 0x0c, + 0xb5, 0xea, 0x3d, 0x3e, 0xaa, 0x81, 0xde, 0x14, 0xbf, 0xd1, 0x3d, 0x98, 0x6a, 0xd7, 0x77, 0x71, + 0xdb, 0xca, 0xa7, 0x2e, 0xa6, 0x8a, 0x99, 0xf2, 0xf5, 0x51, 0x22, 0x2b, 0xdd, 0xa5, 0xa6, 0xef, + 0x77, 0x6c, 0xf3, 0x48, 0xe3, 0xf3, 0xa0, 0x1a, 0x64, 0xf6, 0xf1, 0xfe, 0x2e, 0x36, 0xad, 0x4f, + 0xf5, 0xae, 0x95, 0x9f, 0xb8, 0x98, 0x2a, 0xce, 0x95, 0xaf, 0x84, 0x6d, 0xdb, 0x76, 0x17, 0x37, + 0x4a, 0x1f, 0x3a, 0x78, 0xcd, 0x6b, 0x8b, 0xca, 0x30, 0x69, 0x1a, 0x6d, 0x6c, 0xe5, 0x27, 0xe9, + 0x24, 0x17, 0x42, 0xf7, 0xde, 0x68, 0x63, 0x8d, 0x41, 0xd1, 0x25, 0xc8, 0x92, 0xad, 0x70, 0xf7, + 0x60, 0x8a, 0xee, 0xcf, 0x2c, 0x19, 0x14, 0xab, 0x56, 0xde, 0x84, 0x8c, 0x27, 0x74, 0xb4, 0x00, + 0xa9, 0x16, 0x3e, 0x62, 0xb4, 0xd0, 0xc8, 0x4f, 0xb2, 0xbb, 0x87, 0xf5, 0xf6, 0x01, 0xce, 0x27, + 0xe9, 0x18, 0x7b, 0xb8, 0x91, 0xbc, 0x9e, 0x50, 0x37, 0x60, 0xd1, 0xb3, 0x1d, 0x9c, 0x23, 0x25, + 0x98, 0x24, 0xd9, 0x67, 0xc9, 0x88, 0x22, 0x09, 0x83, 0xa9, 0x3f, 0x24, 0x60, 0xf1, 0x41, 0xb7, + 0x59, 0xb7, 0xf1, 0xa8, 0x0c, 0x45, 0xef, 0xc0, 0x2c, 0x05, 0x1d, 0x62, 0xd3, 0xd2, 0x8d, 0x0e, + 0x0d, 0x30, 0x53, 0x3e, 0x2f, 0xf3, 0xf8, 0x31, 0x83, 0x68, 0x19, 0x62, 0xc0, 0x1f, 0xd0, 0xab, + 0x30, 0x41, 0x8e, 0x5b, 0x3e, 0x45, 0xed, 0x2e, 0x44, 0xe5, 0x45, 0xa3, 0x48, 0xb5, 0x02, 0xc8, + 0x1b, 0xeb, 0xb1, 0x8e, 0xc5, 0x16, 0x2c, 0x6a, 0x78, 0xdf, 0x38, 0x1c, 0x7d, 0xbd, 0x39, 0x98, + 0x7c, 0x68, 0x98, 0x0d, 0x96, 0x89, 0x69, 0x8d, 0x3d, 0xa8, 0x39, 0x40, 0xde, 0xf9, 0x58, 0x4c, + 0xfc, 0xd0, 0xdf, 0xaf, 0x5b, 0x2d, 0x8f, 0x0b, 0xbb, 0x6e, 0xb5, 0x02, 0x2e, 0x08, 0x82, 0xb8, + 0x20, 0xaf, 0x9c, 0x43, 0xcf, 0xcc, 0xdc, 0xd5, 0x91, 0x97, 0x51, 0xab, 0xa3, 0x78, 0x8a, 0x52, + 0xaf, 0x8b, 0xd5, 0x8d, 0xec, 0xda, 0x59, 0x87, 0xd7, 0xbb, 0xfa, 0x2f, 0x17, 0x11, 0x32, 0x78, + 0x0c, 0x11, 0xf1, 0x9a, 0x0d, 0x8a, 0xc8, 0xf7, 0xa7, 0x28, 0x22, 0xb2, 0xc8, 0xa4, 0x22, 0xb2, + 0x0e, 0x19, 0x0b, 0x9b, 0x87, 0x7a, 0x83, 0xb0, 0x83, 0x89, 0x08, 0x0f, 0x61, 0x9b, 0x0d, 0xd7, + 0xaa, 0x96, 0x06, 0x1c, 0x52, 0x6b, 0x5a, 0x68, 0x05, 0xa6, 0x39, 0x97, 0x98, 0x5a, 0xcc, 0x54, + 0x32, 0xfd, 0xde, 0x72, 0x9a, 0x91, 0xc9, 0xd2, 0xd2, 0x8c, 0x4d, 0x16, 0xaa, 0xc2, 0x5c, 0x13, + 0x5b, 0xba, 0x89, 0x9b, 0x3b, 0x96, 0x5d, 0xb7, 0xb9, 0x3e, 0xcc, 0x95, 0x5f, 0x0c, 0x4b, 0xf1, + 0x36, 0x41, 0x69, 0x59, 0x6e, 0x44, 0x9f, 0x24, 0x22, 0x93, 0xfe, 0x5f, 0x44, 0x86, 0x6f, 0x97, + 0x2b, 0x32, 0x84, 0x35, 0x91, 0x22, 0x43, 0x69, 0xc4, 0x60, 0xea, 0x26, 0xe4, 0x36, 0x4c, 0x5c, + 0xb7, 0x31, 0xdf, 0x32, 0x41, 0xa4, 0xab, 0x5c, 0x01, 0x18, 0x8b, 0x96, 0x65, 0xd3, 0x70, 0x0b, + 0x8f, 0x08, 0x6c, 0xc1, 0xd9, 0xc0, 0x64, 0x3c, 0xaa, 0x6b, 0x90, 0xe6, 0x69, 0xe0, 0x13, 0x9e, + 0x8f, 0x98, 0x50, 0x13, 0x58, 0xf5, 0x26, 0x2c, 0xde, 0xc6, 0x76, 0x20, 0xb2, 0x35, 0x00, 0x37, + 0xeb, 0xfc, 0xd4, 0x64, 0xfb, 0xbd, 0xe5, 0x19, 0x27, 0xe9, 0xda, 0x8c, 0x93, 0x73, 0x75, 0x13, + 0x90, 0x77, 0x8a, 0xf1, 0xe2, 0xf9, 0x25, 0x01, 0x39, 0xa6, 0x72, 0xe3, 0xc4, 0x84, 0xaa, 0x30, + 0x2f, 0xd0, 0x23, 0x08, 0xf4, 0x1c, 0xb7, 0x11, 0x1a, 0x7d, 0xd5, 0xa7, 0xd1, 0xc3, 0x67, 0x28, + 0xb0, 0x80, 0xf1, 0x76, 0xa4, 0x0a, 0x39, 0x26, 0x4d, 0x63, 0x25, 0xe9, 0x05, 0x38, 0x1b, 0x98, + 0x85, 0x6b, 0xdc, 0x5f, 0x49, 0x58, 0x22, 0x1c, 0xe7, 0xe3, 0x8e, 0xcc, 0xd5, 0x82, 0x32, 0xb7, + 0x1e, 0x26, 0x26, 0x01, 0xcb, 0x41, 0xa5, 0xfb, 0x26, 0x79, 0xe2, 0x4a, 0xb7, 0x1d, 0x50, 0xba, + 0xb7, 0x46, 0x0c, 0x4e, 0x2a, 0x76, 0x03, 0x6a, 0x32, 0x71, 0xb2, 0x6a, 0xf2, 0x11, 0xe4, 0xfc, + 0x21, 0x71, 0x62, 0xbc, 0x01, 0xd3, 0x3c, 0x51, 0x42, 0x53, 0x22, 0x99, 0xe1, 0x80, 0x5d, 0x65, + 0xd9, 0xc2, 0xf6, 0x63, 0xc3, 0x6c, 0x8d, 0xa0, 0x2c, 0xdc, 0x42, 0xa6, 0x2c, 0xce, 0x64, 0x2e, + 0x6f, 0x3b, 0x6c, 0x28, 0x8a, 0xb7, 0xc2, 0x4a, 0x60, 0xd5, 0x07, 0x54, 0x59, 0x02, 0x91, 0x21, + 0x98, 0x20, 0xbb, 0xc9, 0xf7, 0x8b, 0xfe, 0x26, 0x44, 0xe6, 0x36, 0x84, 0xc8, 0x49, 0x97, 0xc8, + 0xdc, 0x96, 0x10, 0x99, 0x03, 0x1c, 0xb5, 0x39, 0xa1, 0x18, 0x3f, 0x11, 0x67, 0xeb, 0xc4, 0xc3, + 0x74, 0xce, 0x5b, 0x20, 0x52, 0xe7, 0xbc, 0xf1, 0xf1, 0x63, 0x9c, 0xb7, 0x80, 0xe5, 0xf3, 0x75, + 0xde, 0x42, 0x82, 0x3b, 0xcd, 0xf3, 0xe6, 0x86, 0xe4, 0x9e, 0x37, 0x9e, 0xa8, 0xc8, 0xf3, 0x26, + 0x32, 0xe7, 0x80, 0xf9, 0xc7, 0x72, 0xa3, 0x7d, 0x60, 0xd9, 0xd8, 0xf4, 0xe8, 0x70, 0x83, 0x8d, + 0x04, 0x74, 0x98, 0xe3, 0x08, 0x2f, 0x38, 0xc0, 0xa1, 0xaf, 0x33, 0x85, 0x4b, 0x5f, 0x0e, 0x89, + 0xa2, 0xaf, 0xb0, 0x12, 0x58, 0x87, 0x4b, 0xfc, 0xc5, 0x31, 0xb8, 0x14, 0xb0, 0x7c, 0xbe, 0xb8, + 0x14, 0x12, 0xdc, 0x69, 0x72, 0xc9, 0x0d, 0xc9, 0xe5, 0x12, 0xcf, 0x46, 0x24, 0x97, 0x44, 0xea, + 0x1c, 0xb0, 0xfa, 0x6d, 0x02, 0x32, 0x9b, 0xf8, 0x48, 0x33, 0xec, 0xba, 0x4d, 0x6a, 0x8d, 0x97, + 0x61, 0x91, 0x90, 0x0c, 0x9b, 0x3b, 0x8f, 0x0c, 0xbd, 0xb3, 0x63, 0x1b, 0x2d, 0xdc, 0xa1, 0xa1, + 0x4d, 0x6b, 0xf3, 0xec, 0xc5, 0x1d, 0x43, 0xef, 0xdc, 0x27, 0xc3, 0x68, 0x0d, 0xd0, 0x7e, 0xbd, + 0x53, 0xdf, 0xf3, 0x83, 0xd9, 0xc5, 0x6c, 0x81, 0xbf, 0x91, 0xa2, 0x0f, 0x3a, 0x6d, 0xa3, 0xd1, + 0xda, 0x21, 0xab, 0x4e, 0xf9, 0xd0, 0x0f, 0xe8, 0x8b, 0x4d, 0x7c, 0xa4, 0x7e, 0x95, 0x14, 0x05, + 0xd8, 0x38, 0x3c, 0x27, 0x05, 0x98, 0x40, 0x8f, 0x52, 0x80, 0x71, 0x9b, 0x11, 0x0a, 0x30, 0xee, + 0xdd, 0xfd, 0x90, 0xa1, 0x9b, 0x30, 0x6d, 0xf2, 0x5d, 0xcd, 0x4f, 0x84, 0x1b, 0x7a, 0x36, 0xbf, + 0x32, 0xf1, 0xb4, 0xb7, 0x7c, 0x46, 0x73, 0xcc, 0xdc, 0x1a, 0xee, 0x84, 0x0e, 0xea, 0xdb, 0xb0, + 0x40, 0x4b, 0xe4, 0x86, 0x89, 0x6d, 0xb1, 0x9f, 0xab, 0x30, 0x63, 0xd1, 0x01, 0x77, 0x3b, 0x67, + 0xfb, 0xbd, 0xe5, 0x69, 0x86, 0xaa, 0x55, 0xc9, 0x77, 0x9e, 0xfe, 0x6a, 0xaa, 0xb7, 0x79, 0x91, + 0xce, 0xcc, 0x79, 0x28, 0x65, 0x98, 0x62, 0x00, 0x1e, 0x89, 0x22, 0xaf, 0x19, 0xa8, 0x0d, 0x47, + 0xaa, 0x3f, 0x27, 0x60, 0x49, 0x14, 0xa7, 0xc7, 0x8b, 0x05, 0x55, 0x60, 0x8e, 0x43, 0x47, 0xc8, + 0x6b, 0x96, 0x99, 0x88, 0xb4, 0x96, 0x7d, 0x69, 0x2d, 0x84, 0x07, 0xee, 0x29, 0x4f, 0xee, 0xb8, + 0xf7, 0x82, 0xb1, 0xb7, 0xe1, 0xcf, 0x24, 0x20, 0x56, 0x89, 0x91, 0x47, 0x47, 0x36, 0x3f, 0x08, + 0xca, 0x66, 0x29, 0xbc, 0xaa, 0xf4, 0x1a, 0x0e, 0xaa, 0xe6, 0xd7, 0x27, 0xaf, 0x9a, 0x5a, 0x40, + 0x35, 0x6f, 0x8c, 0x16, 0xdb, 0xa9, 0x88, 0xe6, 0xa6, 0xb8, 0x5a, 0xf0, 0x88, 0x78, 0xca, 0x5e, + 0x27, 0x17, 0x21, 0x3a, 0xc4, 0x25, 0x33, 0x2a, 0x67, 0x02, 0xaa, 0xd6, 0x60, 0x49, 0xdc, 0x7c, + 0xbd, 0xd4, 0x2d, 0xfb, 0x6a, 0xdd, 0xa1, 0xb9, 0xe4, 0x9f, 0x6a, 0x0c, 0x2e, 0xbd, 0x07, 0x4b, + 0xe2, 0x62, 0x75, 0xcc, 0xd3, 0x7d, 0xce, 0xbd, 0xe0, 0x79, 0xa3, 0x29, 0xff, 0x78, 0x0e, 0xd2, + 0x1b, 0xac, 0x69, 0x8f, 0x74, 0x48, 0xf3, 0x7e, 0x38, 0x52, 0x65, 0x41, 0xf9, 0x7b, 0xec, 0xca, + 0xa5, 0x48, 0x0c, 0xaf, 0x44, 0xcf, 0xfe, 0xfa, 0xd3, 0xdf, 0xdf, 0x25, 0xe7, 0x21, 0x4b, 0x41, + 0xaf, 0xf0, 0x2f, 0x01, 0x32, 0x60, 0xc6, 0x69, 0xac, 0xa2, 0x97, 0x86, 0x69, 0x43, 0x2b, 0x97, + 0x63, 0x50, 0xd1, 0x0e, 0x4d, 0x00, 0xb7, 0xaf, 0x89, 0xa4, 0x73, 0x0d, 0xf4, 0x68, 0x95, 0x95, + 0x38, 0x58, 0xac, 0x4f, 0xb7, 0x6f, 0x29, 0xf7, 0x39, 0xd0, 0x27, 0x95, 0xfb, 0x94, 0xb4, 0x3f, + 0x43, 0x7c, 0xb2, 0x1c, 0xde, 0xaf, 0x5b, 0xad, 0xd0, 0x1c, 0x7a, 0xfa, 0x96, 0xa1, 0x39, 0xf4, + 0x75, 0x28, 0xa3, 0x73, 0x48, 0xfb, 0x56, 0xe1, 0x39, 0xf4, 0x76, 0x01, 0xc3, 0x73, 0xe8, 0x6b, + 0x7e, 0xc5, 0xee, 0x27, 0x5d, 0x5e, 0xc4, 0x7e, 0x7a, 0x57, 0xb8, 0x12, 0x07, 0x8b, 0xf5, 0xe9, + 0xf6, 0x9d, 0xe4, 0x3e, 0x07, 0x5a, 0x5b, 0x72, 0x9f, 0x83, 0xed, 0xab, 0x30, 0x9f, 0x4f, 0x60, + 0xd6, 0x7b, 0x85, 0x47, 0x57, 0x86, 0xec, 0x3b, 0x28, 0xc5, 0x78, 0x60, 0xb4, 0xe7, 0xcf, 0x21, + 0xeb, 0x6b, 0xfc, 0x21, 0xe9, 0x8c, 0xb2, 0x46, 0xa3, 0xb2, 0x3a, 0x04, 0x32, 0xd6, 0xb9, 0xaf, + 0xa7, 0x25, 0x77, 0x2e, 0xeb, 0xdb, 0xc9, 0x9d, 0x4b, 0x1b, 0x64, 0x11, 0xce, 0x7d, 0xad, 0x2b, + 0xb9, 0x73, 0x59, 0x8f, 0x4c, 0xee, 0x5c, 0xde, 0x07, 0x8b, 0x24, 0x19, 0xbf, 0x0a, 0x86, 0x92, + 0xcc, 0xdf, 0x3e, 0x08, 0x25, 0x59, 0xb0, 0x17, 0x10, 0x4d, 0x32, 0x71, 0x6f, 0x0d, 0x27, 0x59, + 0xe0, 0xb2, 0x1d, 0x4e, 0xb2, 0xe0, 0x15, 0x38, 0x96, 0x64, 0x62, 0xc1, 0x11, 0x24, 0x0b, 0xac, + 0x79, 0x75, 0x08, 0xe4, 0x90, 0x79, 0x8e, 0x74, 0x2e, 0xeb, 0xd7, 0x44, 0xe5, 0x79, 0x48, 0xe7, + 0x2c, 0xcf, 0xbc, 0x70, 0x0f, 0xcd, 0xb3, 0xff, 0x4a, 0x14, 0x9a, 0xe7, 0xc0, 0xad, 0x21, 0x26, + 0xcf, 0xe2, 0x4e, 0x19, 0x9e, 0xe7, 0xc0, 0x45, 0x38, 0x3c, 0xcf, 0xc1, 0xeb, 0x69, 0xec, 0x79, + 0x16, 0x0b, 0x8e, 0x38, 0xcf, 0x81, 0x35, 0xaf, 0x0e, 0x81, 0x8c, 0xfd, 0x38, 0x39, 0xb7, 0x19, + 0xf9, 0xc7, 0x29, 0x78, 0x57, 0x52, 0x2e, 0xc7, 0xa0, 0x62, 0xf7, 0xd9, 0x7b, 0x75, 0x90, 0xef, + 0xb3, 0xe4, 0x5a, 0xa4, 0x14, 0xe3, 0x81, 0xd1, 0x9e, 0x0f, 0x20, 0xe3, 0x29, 0x80, 0xd1, 0xca, + 0x70, 0x35, 0xbb, 0x72, 0x25, 0x16, 0x17, 0xbb, 0x60, 0x6f, 0x7d, 0x2b, 0x5f, 0xb0, 0xa4, 0x98, + 0x56, 0x8a, 0xf1, 0xc0, 0x58, 0xcf, 0xde, 0x5a, 0x56, 0xee, 0x59, 0x52, 0x2f, 0x2b, 0xc5, 0x78, + 0x60, 0xa4, 0xe7, 0xca, 0x85, 0xa7, 0xcf, 0x0a, 0x67, 0x7e, 0x7f, 0x56, 0x38, 0xf3, 0xcf, 0xb3, + 0x42, 0xe2, 0x8b, 0x7e, 0x21, 0xf1, 0xb4, 0x5f, 0x48, 0xfc, 0xd6, 0x2f, 0x24, 0xfe, 0xe8, 0x17, + 0x12, 0xbb, 0x53, 0xf4, 0xbf, 0x51, 0xae, 0xfe, 0x17, 0x00, 0x00, 0xff, 0xff, 0x0c, 0xe8, 0xa4, + 0xf9, 0x06, 0x23, 0x00, 0x00, } diff --git a/api/control.proto b/api/control.proto index a05c55b4ad..e0b37a9156 100644 --- a/api/control.proto +++ b/api/control.proto @@ -313,12 +313,17 @@ message ListClustersResponse { repeated Cluster clusters = 1; } -message JoinTokenRotation { - // RotateWorkerToken tells UpdateCluster to rotate the worker secret. - bool rotate_worker_token = 1; +// KeyRotation tells UpdateCluster what items to rotate +message KeyRotation { + // WorkerJoinToken tells UpdateCluster to rotate the worker secret token. + bool worker_join_token = 1; + + // ManagerJoinToken tells UpdateCluster to rotate the manager secret token. + bool manager_join_token = 2; + + // ManagerUnlockKey tells UpdateCluster to rotate the manager unlock key + bool manager_unlock_key = 3; - // RotateManagerSecret tells UpdateCluster to rotate the manager secret. - bool rotate_manager_token = 2; } message UpdateClusterRequest { @@ -331,8 +336,8 @@ message UpdateClusterRequest { // Spec is the new spec to apply to the cluster. ClusterSpec spec = 3; - // Rotation contains flags for join token rotation - JoinTokenRotation rotation = 4 [(gogoproto.nullable) = false]; + // Rotation contains flags for join token and unlock key rotation + KeyRotation rotation = 4 [(gogoproto.nullable) = false]; } message UpdateClusterResponse { diff --git a/api/objects.pb.go b/api/objects.pb.go index fc54f1b74b..1e6cb0da09 100644 --- a/api/objects.pb.go +++ b/api/objects.pb.go @@ -232,6 +232,12 @@ type Cluster struct { // be honored. It's a mapping from CN -> BlacklistedCertificate. // swarm. Their certificates should effectively be blacklisted. BlacklistedCertificates map[string]*BlacklistedCertificate `protobuf:"bytes,8,rep,name=blacklisted_certificates,json=blacklistedCertificates" json:"blacklisted_certificates,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` + // UnlockKeys defines the keys that lock node data at rest. For example, + // this would contain the key encrypting key (KEK) that will encrypt the + // manager TLS keys at rest and the raft encryption keys at rest. + // If the key is empty, the node will be unlocked (will not require a key + // to start up from a shut down state). + UnlockKeys []*EncryptionKey `protobuf:"bytes,9,rep,name=unlock_keys,json=unlockKeys" json:"unlock_keys,omitempty"` } func (m *Cluster) Reset() { *m = Cluster{} } @@ -460,6 +466,13 @@ func (m *Cluster) Copy() *Cluster { } } + if m.UnlockKeys != nil { + o.UnlockKeys = make([]*EncryptionKey, 0, len(m.UnlockKeys)) + for _, v := range m.UnlockKeys { + o.UnlockKeys = append(o.UnlockKeys, v.Copy()) + } + } + return o } @@ -633,7 +646,7 @@ func (this *Cluster) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&api.Cluster{") s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n") s = append(s, "Meta: "+strings.Replace(this.Meta.GoString(), `&`, ``, 1)+",\n") @@ -656,6 +669,9 @@ func (this *Cluster) GoString() string { if this.BlacklistedCertificates != nil { s = append(s, "BlacklistedCertificates: "+mapStringForBlacklistedCertificates+",\n") } + if this.UnlockKeys != nil { + s = append(s, "UnlockKeys: "+fmt.Sprintf("%#v", this.UnlockKeys)+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -1310,6 +1326,18 @@ func (m *Cluster) MarshalTo(data []byte) (int, error) { } } } + if len(m.UnlockKeys) > 0 { + for _, msg := range m.UnlockKeys { + data[i] = 0x4a + i++ + i = encodeVarintObjects(data, i, uint64(msg.Size())) + n, err := msg.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n + } + } return i, nil } @@ -1637,6 +1665,12 @@ func (m *Cluster) Size() (n int) { n += mapEntrySize + 1 + sovObjects(uint64(mapEntrySize)) } } + if len(m.UnlockKeys) > 0 { + for _, e := range m.UnlockKeys { + l = e.Size() + n += 1 + l + sovObjects(uint64(l)) + } + } return n } @@ -1814,6 +1848,7 @@ func (this *Cluster) String() string { `NetworkBootstrapKeys:` + strings.Replace(fmt.Sprintf("%v", this.NetworkBootstrapKeys), "EncryptionKey", "EncryptionKey", 1) + `,`, `EncryptionKeyLamportClock:` + fmt.Sprintf("%v", this.EncryptionKeyLamportClock) + `,`, `BlacklistedCertificates:` + mapStringForBlacklistedCertificates + `,`, + `UnlockKeys:` + strings.Replace(fmt.Sprintf("%v", this.UnlockKeys), "EncryptionKey", "EncryptionKey", 1) + `,`, `}`, }, "") return s @@ -3863,6 +3898,37 @@ func (m *Cluster) Unmarshal(data []byte) error { m.BlacklistedCertificates[mapkey] = mapvalue } iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field UnlockKeys", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowObjects + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthObjects + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.UnlockKeys = append(m.UnlockKeys, &EncryptionKey{}) + if err := m.UnlockKeys[len(m.UnlockKeys)-1].Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipObjects(data[iNdEx:]) @@ -4199,79 +4265,80 @@ var ( func init() { proto.RegisterFile("objects.proto", fileDescriptorObjects) } var fileDescriptorObjects = []byte{ - // 1174 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4d, 0x8f, 0x1b, 0x35, - 0x18, 0xee, 0x24, 0xb3, 0xf9, 0x78, 0xb3, 0x59, 0x81, 0xa9, 0xca, 0x34, 0x2c, 0xc9, 0x92, 0x0a, - 0x54, 0xa1, 0x2a, 0x15, 0xa5, 0xa0, 0x2d, 0xb4, 0x82, 0x7c, 0x09, 0xa2, 0x52, 0xa8, 0xdc, 0xd2, - 0x1e, 0x23, 0xef, 0x8c, 0x1b, 0x86, 0x4c, 0xc6, 0x23, 0xdb, 0x49, 0x95, 0x9e, 0x10, 0x3f, 0x80, - 0x9f, 0xc0, 0x5f, 0xe1, 0xba, 0x07, 0x0e, 0xdc, 0xe0, 0x80, 0x22, 0x36, 0x07, 0x24, 0x6e, 0xfc, - 0x04, 0x64, 0x8f, 0x27, 0x99, 0x55, 0x26, 0xcb, 0x56, 0xaa, 0xf6, 0xe6, 0x37, 0x7e, 0x9e, 0xc7, - 0xef, 0x97, 0xdf, 0x71, 0xa0, 0xca, 0x8e, 0xbe, 0xa7, 0xae, 0x14, 0xad, 0x88, 0x33, 0xc9, 0x10, - 0xf2, 0x98, 0x3b, 0xa6, 0xbc, 0x25, 0x9e, 0x13, 0x3e, 0x19, 0xfb, 0xb2, 0x35, 0xfb, 0xa0, 0x56, - 0x91, 0xf3, 0x88, 0x1a, 0x40, 0xad, 0x22, 0x22, 0xea, 0x26, 0xc6, 0x55, 0xe9, 0x4f, 0xa8, 0x90, - 0x64, 0x12, 0xdd, 0x5c, 0xad, 0xcc, 0xd6, 0xe5, 0x11, 0x1b, 0x31, 0xbd, 0xbc, 0xa9, 0x56, 0xf1, - 0xaf, 0xcd, 0x5f, 0x2c, 0xb0, 0x1f, 0x50, 0x49, 0xd0, 0xa7, 0x50, 0x9c, 0x51, 0x2e, 0x7c, 0x16, - 0x3a, 0xd6, 0x81, 0x75, 0xbd, 0x72, 0xeb, 0xad, 0xd6, 0xe6, 0xc9, 0xad, 0x27, 0x31, 0xa4, 0x63, - 0x1f, 0x2f, 0x1a, 0x97, 0x70, 0xc2, 0x40, 0x77, 0x01, 0x5c, 0x4e, 0x89, 0xa4, 0xde, 0x90, 0x48, - 0x27, 0xa7, 0xf9, 0x6f, 0x67, 0xf1, 0x1f, 0x27, 0x4e, 0xe1, 0xb2, 0x21, 0xb4, 0xa5, 0x62, 0x4f, - 0x23, 0x2f, 0x61, 0xe7, 0xcf, 0xc5, 0x36, 0x84, 0xb6, 0x6c, 0xfe, 0x93, 0x07, 0xfb, 0x6b, 0xe6, - 0x51, 0x74, 0x05, 0x72, 0xbe, 0xa7, 0x9d, 0x2f, 0x77, 0x0a, 0xcb, 0x45, 0x23, 0x37, 0xe8, 0xe1, - 0x9c, 0xef, 0xa1, 0x5b, 0x60, 0x4f, 0xa8, 0x24, 0xc6, 0x2d, 0x27, 0x4b, 0x58, 0x65, 0xc0, 0xc4, - 0xa4, 0xb1, 0xe8, 0x63, 0xb0, 0x55, 0x5a, 0x8d, 0x33, 0xfb, 0x59, 0x1c, 0x75, 0xe6, 0xa3, 0x88, - 0xba, 0x09, 0x4f, 0xe1, 0x51, 0x1f, 0x2a, 0x1e, 0x15, 0x2e, 0xf7, 0x23, 0xa9, 0x32, 0x69, 0x6b, - 0xfa, 0xb5, 0x6d, 0xf4, 0xde, 0x1a, 0x8a, 0xd3, 0x3c, 0x74, 0x17, 0x0a, 0x42, 0x12, 0x39, 0x15, - 0xce, 0x8e, 0x56, 0xa8, 0x6f, 0x75, 0x40, 0xa3, 0x8c, 0x0b, 0x86, 0x83, 0xbe, 0x84, 0xbd, 0x09, - 0x09, 0xc9, 0x88, 0xf2, 0xa1, 0x51, 0x29, 0x68, 0x95, 0x77, 0x32, 0x43, 0x8f, 0x91, 0xb1, 0x10, - 0xae, 0x4e, 0xd2, 0x26, 0xea, 0x03, 0x10, 0x29, 0x89, 0xfb, 0xdd, 0x84, 0x86, 0xd2, 0x29, 0x6a, - 0x95, 0x77, 0x33, 0x7d, 0xa1, 0xf2, 0x39, 0xe3, 0xe3, 0xf6, 0x0a, 0x8c, 0x53, 0x44, 0xf4, 0x05, - 0x54, 0x5c, 0xca, 0xa5, 0xff, 0xcc, 0x77, 0x89, 0xa4, 0x4e, 0x49, 0xeb, 0x34, 0xb2, 0x74, 0xba, - 0x6b, 0x98, 0x09, 0x2a, 0xcd, 0x6c, 0xfe, 0x9e, 0x83, 0xe2, 0x23, 0xca, 0x67, 0xbe, 0xfb, 0x6a, - 0xcb, 0x7d, 0xe7, 0x54, 0xb9, 0x33, 0x3d, 0x33, 0xc7, 0x6e, 0x54, 0xfc, 0x10, 0x4a, 0x34, 0xf4, - 0x22, 0xe6, 0x87, 0xd2, 0x94, 0x3b, 0xb3, 0x5b, 0xfa, 0x06, 0x83, 0x57, 0x68, 0xd4, 0x87, 0x6a, - 0xdc, 0xc5, 0xc3, 0x53, 0xb5, 0x3e, 0xc8, 0xa2, 0x7f, 0xab, 0x81, 0xa6, 0x48, 0xbb, 0xd3, 0x94, - 0x85, 0x7a, 0x50, 0x8d, 0x38, 0x9d, 0xf9, 0x6c, 0x2a, 0x86, 0x3a, 0x88, 0xc2, 0xb9, 0x82, 0xc0, - 0xbb, 0x09, 0x4b, 0x59, 0xcd, 0x9f, 0x73, 0x50, 0x4a, 0x7c, 0x44, 0xb7, 0x4d, 0x3a, 0xac, 0xed, - 0x0e, 0x25, 0x58, 0x2d, 0x15, 0x67, 0xe2, 0x36, 0xec, 0x44, 0x8c, 0x4b, 0xe1, 0xe4, 0x0e, 0xf2, - 0xdb, 0x7a, 0xf6, 0x21, 0xe3, 0xb2, 0xcb, 0xc2, 0x67, 0xfe, 0x08, 0xc7, 0x60, 0xf4, 0x14, 0x2a, - 0x33, 0x9f, 0xcb, 0x29, 0x09, 0x86, 0x7e, 0x24, 0x9c, 0xbc, 0xe6, 0xbe, 0x77, 0xd6, 0x91, 0xad, - 0x27, 0x31, 0x7e, 0xf0, 0xb0, 0xb3, 0xb7, 0x5c, 0x34, 0x60, 0x65, 0x0a, 0x0c, 0x46, 0x6a, 0x10, - 0x89, 0xda, 0x03, 0x28, 0xaf, 0x76, 0xd0, 0x0d, 0x80, 0x30, 0x6e, 0xd1, 0xe1, 0xaa, 0x69, 0xaa, - 0xcb, 0x45, 0xa3, 0x6c, 0x1a, 0x77, 0xd0, 0xc3, 0x65, 0x03, 0x18, 0x78, 0x08, 0x81, 0x4d, 0x3c, - 0x8f, 0xeb, 0x16, 0x2a, 0x63, 0xbd, 0x6e, 0xfe, 0xba, 0x03, 0xf6, 0x63, 0x22, 0xc6, 0x17, 0x3d, - 0x66, 0xd4, 0x99, 0x1b, 0x4d, 0x77, 0x03, 0x40, 0xc4, 0xa5, 0x54, 0xe1, 0xd8, 0xeb, 0x70, 0x4c, - 0x81, 0x55, 0x38, 0x06, 0x10, 0x87, 0x23, 0x02, 0x26, 0x75, 0x7f, 0xd9, 0x58, 0xaf, 0xd1, 0x35, - 0x28, 0x86, 0xcc, 0xd3, 0xf4, 0x82, 0xa6, 0xc3, 0x72, 0xd1, 0x28, 0xa8, 0x91, 0x32, 0xe8, 0xe1, - 0x82, 0xda, 0x1a, 0x78, 0xea, 0xde, 0x92, 0x30, 0x64, 0x92, 0xa8, 0xa1, 0x24, 0xcc, 0xfd, 0xcf, - 0x6c, 0xac, 0xf6, 0x1a, 0x96, 0xdc, 0xdb, 0x14, 0x13, 0x3d, 0x81, 0x37, 0x12, 0x7f, 0xd3, 0x82, - 0xa5, 0x97, 0x11, 0x44, 0x46, 0x21, 0xb5, 0x93, 0x9a, 0x93, 0xe5, 0xed, 0x73, 0x52, 0x67, 0x30, - 0x6b, 0x4e, 0x76, 0xa0, 0xea, 0x51, 0xe1, 0x73, 0xea, 0xe9, 0x1b, 0x48, 0x1d, 0x38, 0xb0, 0xae, - 0xef, 0x6d, 0xf9, 0xf4, 0x18, 0x11, 0x8a, 0x77, 0x0d, 0x47, 0x5b, 0xa8, 0x0d, 0x25, 0xd3, 0x37, - 0xc2, 0xa9, 0xe8, 0xde, 0x3d, 0xe7, 0x7c, 0x5c, 0xd1, 0x4e, 0x4d, 0x90, 0xdd, 0x97, 0x9a, 0x20, - 0x77, 0x00, 0x02, 0x36, 0x1a, 0x7a, 0xdc, 0x9f, 0x51, 0xee, 0x54, 0x35, 0xb7, 0x96, 0xc5, 0xed, - 0x69, 0x04, 0x2e, 0x07, 0x6c, 0x14, 0x2f, 0x9b, 0x3f, 0x5a, 0xf0, 0xfa, 0x86, 0x53, 0xe8, 0x23, - 0x28, 0x1a, 0xb7, 0xce, 0x7a, 0x04, 0x18, 0x1e, 0x4e, 0xb0, 0x68, 0x1f, 0xca, 0xea, 0x8e, 0x50, - 0x21, 0x68, 0x7c, 0xfb, 0xcb, 0x78, 0xfd, 0x03, 0x72, 0xa0, 0x48, 0x02, 0x9f, 0xa8, 0xbd, 0xbc, - 0xde, 0x4b, 0xcc, 0xe6, 0x4f, 0x39, 0x28, 0x1a, 0xb1, 0x8b, 0x1e, 0xe7, 0xe6, 0xd8, 0x8d, 0x9b, - 0x75, 0x0f, 0x76, 0xe3, 0x74, 0x9a, 0x96, 0xb0, 0xff, 0x37, 0xa9, 0x95, 0x18, 0x1f, 0xb7, 0xc3, - 0x3d, 0xb0, 0xfd, 0x88, 0x4c, 0xcc, 0x28, 0xcf, 0x3c, 0x79, 0xf0, 0xb0, 0xfd, 0xe0, 0x9b, 0x28, - 0xee, 0xec, 0xd2, 0x72, 0xd1, 0xb0, 0xd5, 0x0f, 0x58, 0xd3, 0x9a, 0x7f, 0xda, 0x50, 0xec, 0x06, - 0x53, 0x21, 0x29, 0xbf, 0xe8, 0x84, 0x98, 0x63, 0x37, 0x12, 0xd2, 0x85, 0x22, 0x67, 0x4c, 0x0e, - 0x5d, 0x72, 0x56, 0x2e, 0x30, 0x63, 0xb2, 0xdb, 0xee, 0xec, 0x29, 0xa2, 0x1a, 0x24, 0xb1, 0x8d, - 0x0b, 0x8a, 0xda, 0x25, 0xe8, 0x29, 0x5c, 0x49, 0xc6, 0xef, 0x11, 0x63, 0x52, 0x48, 0x4e, 0xa2, - 0xe1, 0x98, 0xce, 0xd5, 0x37, 0x2f, 0xbf, 0xed, 0x65, 0xd2, 0x0f, 0x5d, 0x3e, 0xd7, 0x89, 0xba, - 0x4f, 0xe7, 0xf8, 0xb2, 0x11, 0xe8, 0x24, 0xfc, 0xfb, 0x74, 0x2e, 0xd0, 0x67, 0xb0, 0x4f, 0x57, - 0x30, 0xa5, 0x38, 0x0c, 0xc8, 0x44, 0x7d, 0x58, 0x86, 0x6e, 0xc0, 0xdc, 0xb1, 0x9e, 0x6d, 0x36, - 0xbe, 0x4a, 0xd3, 0x52, 0x5f, 0xc5, 0x88, 0xae, 0x02, 0x20, 0x01, 0xce, 0x51, 0x40, 0xdc, 0x71, - 0xe0, 0x0b, 0xf5, 0xfe, 0x4c, 0x3d, 0x36, 0xd4, 0x78, 0x52, 0xbe, 0x1d, 0x9e, 0x91, 0xad, 0x56, - 0x67, 0xcd, 0x4d, 0x3d, 0x5d, 0x44, 0x3f, 0x94, 0x7c, 0x8e, 0xdf, 0x3c, 0xca, 0xde, 0xad, 0xcd, - 0x60, 0xff, 0x2c, 0x22, 0x7a, 0x0d, 0xf2, 0x63, 0x3a, 0x8f, 0x6b, 0x8f, 0xd5, 0x12, 0x7d, 0x0e, - 0x3b, 0x33, 0x12, 0x4c, 0xa9, 0xa9, 0xfa, 0xfb, 0x59, 0x3e, 0x65, 0x4b, 0xe2, 0x98, 0xf8, 0x49, - 0xee, 0xd0, 0x6a, 0xfe, 0x6d, 0x41, 0xe1, 0x11, 0x75, 0x39, 0x95, 0xaf, 0xb4, 0xbb, 0x0e, 0x4f, - 0x75, 0x57, 0x3d, 0xfb, 0xe1, 0xa1, 0x4e, 0xdd, 0x68, 0xae, 0x2b, 0x50, 0xf0, 0xfc, 0x11, 0x15, - 0xf1, 0xd3, 0xa9, 0x8c, 0x8d, 0x85, 0x9a, 0x60, 0x0b, 0xff, 0x05, 0xd5, 0xd7, 0x28, 0x1f, 0x7f, - 0xe5, 0x8d, 0x82, 0xff, 0x82, 0x62, 0xbd, 0x87, 0x6a, 0x50, 0xf2, 0x43, 0x49, 0x79, 0x48, 0x02, - 0x5d, 0xe6, 0x12, 0x5e, 0xd9, 0x9d, 0xfd, 0xe3, 0x93, 0xfa, 0xa5, 0x3f, 0x4e, 0xea, 0x97, 0xfe, - 0x3d, 0xa9, 0x5b, 0x3f, 0x2c, 0xeb, 0xd6, 0xf1, 0xb2, 0x6e, 0xfd, 0xb6, 0xac, 0x5b, 0x7f, 0x2d, - 0xeb, 0xd6, 0x51, 0x41, 0xff, 0xf5, 0xf9, 0xf0, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x40, 0xcf, - 0x57, 0x63, 0x6a, 0x0d, 0x00, 0x00, + // 1192 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xbc, 0x57, 0x4f, 0x6f, 0x1b, 0x45, + 0x14, 0xef, 0xda, 0x1b, 0xdb, 0xfb, 0x1c, 0x47, 0x30, 0x54, 0x65, 0x1b, 0x82, 0x1d, 0x5c, 0x81, + 0x2a, 0x54, 0xb9, 0xa2, 0x14, 0x94, 0x42, 0x2b, 0xb0, 0x9d, 0x08, 0xac, 0x52, 0xa8, 0xa6, 0xa5, + 0x3d, 0x5a, 0x93, 0xdd, 0xa9, 0x59, 0xbc, 0xde, 0x59, 0xcd, 0x8c, 0x5d, 0xb9, 0x27, 0xc4, 0x07, + 0xe0, 0x23, 0x20, 0xbe, 0x09, 0xd7, 0x1e, 0x38, 0x70, 0x83, 0x53, 0x44, 0x7d, 0x40, 0xe2, 0xc6, + 0x47, 0x40, 0xf3, 0x67, 0x9d, 0x8d, 0xbc, 0x0e, 0xa9, 0x54, 0xe5, 0x36, 0xe3, 0xf9, 0xfd, 0x7e, + 0xef, 0xcd, 0x9b, 0xdf, 0xbc, 0x1d, 0x43, 0x83, 0x1d, 0x7e, 0x4f, 0x03, 0x29, 0x3a, 0x29, 0x67, + 0x92, 0x21, 0x14, 0xb2, 0x60, 0x4c, 0x79, 0x47, 0x3c, 0x25, 0x7c, 0x32, 0x8e, 0x64, 0x67, 0xf6, + 0xc1, 0x76, 0x5d, 0xce, 0x53, 0x6a, 0x01, 0xdb, 0x75, 0x91, 0xd2, 0x20, 0x9b, 0x5c, 0x96, 0xd1, + 0x84, 0x0a, 0x49, 0x26, 0xe9, 0xf5, 0xe5, 0xc8, 0x2e, 0x5d, 0x1c, 0xb1, 0x11, 0xd3, 0xc3, 0xeb, + 0x6a, 0x64, 0x7e, 0x6d, 0xff, 0xea, 0x80, 0x7b, 0x8f, 0x4a, 0x82, 0x3e, 0x85, 0xea, 0x8c, 0x72, + 0x11, 0xb1, 0xc4, 0x77, 0x76, 0x9d, 0xab, 0xf5, 0x1b, 0x6f, 0x75, 0x56, 0x23, 0x77, 0x1e, 0x19, + 0x48, 0xcf, 0x7d, 0x7e, 0xd4, 0xba, 0x80, 0x33, 0x06, 0xba, 0x0d, 0x10, 0x70, 0x4a, 0x24, 0x0d, + 0x87, 0x44, 0xfa, 0x25, 0xcd, 0x7f, 0xbb, 0x88, 0xff, 0x30, 0x4b, 0x0a, 0x7b, 0x96, 0xd0, 0x95, + 0x8a, 0x3d, 0x4d, 0xc3, 0x8c, 0x5d, 0x3e, 0x13, 0xdb, 0x12, 0xba, 0xb2, 0xfd, 0x4f, 0x19, 0xdc, + 0xaf, 0x59, 0x48, 0xd1, 0x25, 0x28, 0x45, 0xa1, 0x4e, 0xde, 0xeb, 0x55, 0x16, 0x47, 0xad, 0xd2, + 0x60, 0x1f, 0x97, 0xa2, 0x10, 0xdd, 0x00, 0x77, 0x42, 0x25, 0xb1, 0x69, 0xf9, 0x45, 0xc2, 0xaa, + 0x02, 0x76, 0x4f, 0x1a, 0x8b, 0x3e, 0x06, 0x57, 0x95, 0xd5, 0x26, 0xb3, 0x53, 0xc4, 0x51, 0x31, + 0x1f, 0xa4, 0x34, 0xc8, 0x78, 0x0a, 0x8f, 0x0e, 0xa0, 0x1e, 0x52, 0x11, 0xf0, 0x28, 0x95, 0xaa, + 0x92, 0xae, 0xa6, 0x5f, 0x59, 0x47, 0xdf, 0x3f, 0x86, 0xe2, 0x3c, 0x0f, 0xdd, 0x86, 0x8a, 0x90, + 0x44, 0x4e, 0x85, 0xbf, 0xa1, 0x15, 0x9a, 0x6b, 0x13, 0xd0, 0x28, 0x9b, 0x82, 0xe5, 0xa0, 0x2f, + 0x61, 0x6b, 0x42, 0x12, 0x32, 0xa2, 0x7c, 0x68, 0x55, 0x2a, 0x5a, 0xe5, 0x9d, 0xc2, 0xad, 0x1b, + 0xa4, 0x11, 0xc2, 0x8d, 0x49, 0x7e, 0x8a, 0x0e, 0x00, 0x88, 0x94, 0x24, 0xf8, 0x6e, 0x42, 0x13, + 0xe9, 0x57, 0xb5, 0xca, 0xbb, 0x85, 0xb9, 0x50, 0xf9, 0x94, 0xf1, 0x71, 0x77, 0x09, 0xc6, 0x39, + 0x22, 0xfa, 0x02, 0xea, 0x01, 0xe5, 0x32, 0x7a, 0x12, 0x05, 0x44, 0x52, 0xbf, 0xa6, 0x75, 0x5a, + 0x45, 0x3a, 0xfd, 0x63, 0x98, 0xdd, 0x54, 0x9e, 0xd9, 0xfe, 0xa3, 0x04, 0xd5, 0x07, 0x94, 0xcf, + 0xa2, 0xe0, 0xd5, 0x1e, 0xf7, 0xad, 0x13, 0xc7, 0x5d, 0x98, 0x99, 0x0d, 0xbb, 0x72, 0xe2, 0x7b, + 0x50, 0xa3, 0x49, 0x98, 0xb2, 0x28, 0x91, 0xf6, 0xb8, 0x0b, 0xdd, 0x72, 0x60, 0x31, 0x78, 0x89, + 0x46, 0x07, 0xd0, 0x30, 0x2e, 0x1e, 0x9e, 0x38, 0xeb, 0xdd, 0x22, 0xfa, 0xb7, 0x1a, 0x68, 0x0f, + 0x69, 0x73, 0x9a, 0x9b, 0xa1, 0x7d, 0x68, 0xa4, 0x9c, 0xce, 0x22, 0x36, 0x15, 0x43, 0xbd, 0x89, + 0xca, 0x99, 0x36, 0x81, 0x37, 0x33, 0x96, 0x9a, 0xb5, 0x7f, 0x2e, 0x41, 0x2d, 0xcb, 0x11, 0xdd, + 0xb4, 0xe5, 0x70, 0xd6, 0x27, 0x94, 0x61, 0xb5, 0x94, 0xa9, 0xc4, 0x4d, 0xd8, 0x48, 0x19, 0x97, + 0xc2, 0x2f, 0xed, 0x96, 0xd7, 0x79, 0xf6, 0x3e, 0xe3, 0xb2, 0xcf, 0x92, 0x27, 0xd1, 0x08, 0x1b, + 0x30, 0x7a, 0x0c, 0xf5, 0x59, 0xc4, 0xe5, 0x94, 0xc4, 0xc3, 0x28, 0x15, 0x7e, 0x59, 0x73, 0xdf, + 0x3b, 0x2d, 0x64, 0xe7, 0x91, 0xc1, 0x0f, 0xee, 0xf7, 0xb6, 0x16, 0x47, 0x2d, 0x58, 0x4e, 0x05, + 0x06, 0x2b, 0x35, 0x48, 0xc5, 0xf6, 0x3d, 0xf0, 0x96, 0x2b, 0xe8, 0x1a, 0x40, 0x62, 0x2c, 0x3a, + 0x5c, 0x9a, 0xa6, 0xb1, 0x38, 0x6a, 0x79, 0xd6, 0xb8, 0x83, 0x7d, 0xec, 0x59, 0xc0, 0x20, 0x44, + 0x08, 0x5c, 0x12, 0x86, 0x5c, 0x5b, 0xc8, 0xc3, 0x7a, 0xdc, 0xfe, 0x6d, 0x03, 0xdc, 0x87, 0x44, + 0x8c, 0xcf, 0xbb, 0xcd, 0xa8, 0x98, 0x2b, 0xa6, 0xbb, 0x06, 0x20, 0xcc, 0x51, 0xaa, 0xed, 0xb8, + 0xc7, 0xdb, 0xb1, 0x07, 0xac, 0xb6, 0x63, 0x01, 0x66, 0x3b, 0x22, 0x66, 0x52, 0xfb, 0xcb, 0xc5, + 0x7a, 0x8c, 0xae, 0x40, 0x35, 0x61, 0xa1, 0xa6, 0x57, 0x34, 0x1d, 0x16, 0x47, 0xad, 0x8a, 0x6a, + 0x29, 0x83, 0x7d, 0x5c, 0x51, 0x4b, 0x83, 0x50, 0xdd, 0x5b, 0x92, 0x24, 0x4c, 0x12, 0xd5, 0x94, + 0x84, 0xbd, 0xff, 0x85, 0xc6, 0xea, 0x1e, 0xc3, 0xb2, 0x7b, 0x9b, 0x63, 0xa2, 0x47, 0xf0, 0x46, + 0x96, 0x6f, 0x5e, 0xb0, 0xf6, 0x32, 0x82, 0xc8, 0x2a, 0xe4, 0x56, 0x72, 0x7d, 0xd2, 0x5b, 0xdf, + 0x27, 0x75, 0x05, 0x8b, 0xfa, 0x64, 0x0f, 0x1a, 0x21, 0x15, 0x11, 0xa7, 0xa1, 0xbe, 0x81, 0xd4, + 0x87, 0x5d, 0xe7, 0xea, 0xd6, 0x9a, 0x4f, 0x8f, 0x15, 0xa1, 0x78, 0xd3, 0x72, 0xf4, 0x0c, 0x75, + 0xa1, 0x66, 0x7d, 0x23, 0xfc, 0xba, 0xf6, 0xee, 0x19, 0xfb, 0xe3, 0x92, 0x76, 0xa2, 0x83, 0x6c, + 0xbe, 0x54, 0x07, 0xb9, 0x05, 0x10, 0xb3, 0xd1, 0x30, 0xe4, 0xd1, 0x8c, 0x72, 0xbf, 0xa1, 0xb9, + 0xdb, 0x45, 0xdc, 0x7d, 0x8d, 0xc0, 0x5e, 0xcc, 0x46, 0x66, 0xd8, 0xfe, 0xd1, 0x81, 0xd7, 0x57, + 0x92, 0x42, 0x1f, 0x41, 0xd5, 0xa6, 0x75, 0xda, 0x23, 0xc0, 0xf2, 0x70, 0x86, 0x45, 0x3b, 0xe0, + 0xa9, 0x3b, 0x42, 0x85, 0xa0, 0xe6, 0xf6, 0x7b, 0xf8, 0xf8, 0x07, 0xe4, 0x43, 0x95, 0xc4, 0x11, + 0x51, 0x6b, 0x65, 0xbd, 0x96, 0x4d, 0xdb, 0x3f, 0x95, 0xa0, 0x6a, 0xc5, 0xce, 0xbb, 0x9d, 0xdb, + 0xb0, 0x2b, 0x37, 0xeb, 0x0e, 0x6c, 0x9a, 0x72, 0x5a, 0x4b, 0xb8, 0xff, 0x5b, 0xd4, 0xba, 0xc1, + 0x1b, 0x3b, 0xdc, 0x01, 0x37, 0x4a, 0xc9, 0xc4, 0xb6, 0xf2, 0xc2, 0xc8, 0x83, 0xfb, 0xdd, 0x7b, + 0xdf, 0xa4, 0xc6, 0xd9, 0xb5, 0xc5, 0x51, 0xcb, 0x55, 0x3f, 0x60, 0x4d, 0x6b, 0xff, 0xb2, 0x01, + 0xd5, 0x7e, 0x3c, 0x15, 0x92, 0xf2, 0xf3, 0x2e, 0x88, 0x0d, 0xbb, 0x52, 0x90, 0x3e, 0x54, 0x39, + 0x63, 0x72, 0x18, 0x90, 0xd3, 0x6a, 0x81, 0x19, 0x93, 0xfd, 0x6e, 0x6f, 0x4b, 0x11, 0x55, 0x23, + 0x31, 0x73, 0x5c, 0x51, 0xd4, 0x3e, 0x41, 0x8f, 0xe1, 0x52, 0xd6, 0x7e, 0x0f, 0x19, 0x93, 0x42, + 0x72, 0x92, 0x0e, 0xc7, 0x74, 0xae, 0xbe, 0x79, 0xe5, 0x75, 0x2f, 0x93, 0x83, 0x24, 0xe0, 0x73, + 0x5d, 0xa8, 0xbb, 0x74, 0x8e, 0x2f, 0x5a, 0x81, 0x5e, 0xc6, 0xbf, 0x4b, 0xe7, 0x02, 0x7d, 0x06, + 0x3b, 0x74, 0x09, 0x53, 0x8a, 0xc3, 0x98, 0x4c, 0xd4, 0x87, 0x65, 0x18, 0xc4, 0x2c, 0x18, 0xeb, + 0xde, 0xe6, 0xe2, 0xcb, 0x34, 0x2f, 0xf5, 0x95, 0x41, 0xf4, 0x15, 0x00, 0x09, 0xf0, 0x0f, 0x63, + 0x12, 0x8c, 0xe3, 0x48, 0xa8, 0xf7, 0x67, 0xee, 0xb1, 0xa1, 0xda, 0x93, 0xca, 0x6d, 0xef, 0x94, + 0x6a, 0x75, 0x7a, 0xc7, 0xdc, 0xdc, 0xd3, 0x45, 0x1c, 0x24, 0x92, 0xcf, 0xf1, 0x9b, 0x87, 0xc5, + 0xab, 0xa8, 0x07, 0xf5, 0x69, 0xa2, 0xc2, 0x9b, 0x1a, 0x78, 0x67, 0xad, 0x01, 0x18, 0x96, 0xda, + 0xf9, 0xf6, 0x0c, 0x76, 0x4e, 0x0b, 0x8e, 0x5e, 0x83, 0xf2, 0x98, 0xce, 0x8d, 0x7f, 0xb0, 0x1a, + 0xa2, 0xcf, 0x61, 0x63, 0x46, 0xe2, 0x29, 0xb5, 0xce, 0x79, 0xbf, 0x28, 0x5e, 0xb1, 0x24, 0x36, + 0xc4, 0x4f, 0x4a, 0x7b, 0x4e, 0xfb, 0x6f, 0x07, 0x2a, 0x0f, 0x68, 0xc0, 0xa9, 0x7c, 0xa5, 0x0e, + 0xdd, 0x3b, 0xe1, 0xd0, 0x66, 0xf1, 0xe3, 0x45, 0x45, 0x5d, 0x31, 0xe8, 0x25, 0xa8, 0x84, 0xd1, + 0x88, 0x0a, 0xf3, 0xfc, 0xf2, 0xb0, 0x9d, 0xa1, 0x36, 0xb8, 0x22, 0x7a, 0x46, 0xf5, 0x55, 0x2c, + 0x9b, 0x97, 0x82, 0x55, 0x88, 0x9e, 0x51, 0xac, 0xd7, 0xd0, 0x36, 0xd4, 0xa2, 0x44, 0x52, 0x9e, + 0x90, 0x58, 0x5b, 0xa5, 0x86, 0x97, 0xf3, 0xde, 0xce, 0xf3, 0x17, 0xcd, 0x0b, 0x7f, 0xbe, 0x68, + 0x5e, 0xf8, 0xf7, 0x45, 0xd3, 0xf9, 0x61, 0xd1, 0x74, 0x9e, 0x2f, 0x9a, 0xce, 0xef, 0x8b, 0xa6, + 0xf3, 0xd7, 0xa2, 0xe9, 0x1c, 0x56, 0xf4, 0xdf, 0xa7, 0x0f, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x49, 0x8f, 0xcd, 0x22, 0xae, 0x0d, 0x00, 0x00, } diff --git a/api/objects.proto b/api/objects.proto index 86421c0851..59f2c267ca 100644 --- a/api/objects.proto +++ b/api/objects.proto @@ -230,6 +230,13 @@ message Cluster { // be honored. It's a mapping from CN -> BlacklistedCertificate. // swarm. Their certificates should effectively be blacklisted. map blacklisted_certificates = 8; + + // UnlockKeys defines the keys that lock node data at rest. For example, + // this would contain the key encrypting key (KEK) that will encrypt the + // manager TLS keys at rest and the raft encryption keys at rest. + // If the key is empty, the node will be unlocked (will not require a key + // to start up from a shut down state). + repeated EncryptionKey unlock_keys = 9; } // Secret represents a secret that should be passed to a container or a node, diff --git a/api/specs.pb.go b/api/specs.pb.go index 97eeca2f8f..30f573e6df 100644 --- a/api/specs.pb.go +++ b/api/specs.pb.go @@ -596,6 +596,8 @@ type ClusterSpec struct { CAConfig CAConfig `protobuf:"bytes,6,opt,name=ca_config,json=caConfig" json:"ca_config"` // TaskDefaults specifies the default values to use for task creation. TaskDefaults TaskDefaults `protobuf:"bytes,7,opt,name=task_defaults,json=taskDefaults" json:"task_defaults"` + // EncryptionConfig defines the cluster's encryption settings. + EncryptionConfig EncryptionConfig `protobuf:"bytes,8,opt,name=encryption_config,json=encryptionConfig" json:"encryption_config"` } func (m *ClusterSpec) Reset() { *m = ClusterSpec{} } @@ -908,6 +910,7 @@ func (m *ClusterSpec) Copy() *ClusterSpec { Dispatcher: *m.Dispatcher.Copy(), CAConfig: *m.CAConfig.Copy(), TaskDefaults: *m.TaskDefaults.Copy(), + EncryptionConfig: *m.EncryptionConfig.Copy(), } return o @@ -1159,7 +1162,7 @@ func (this *ClusterSpec) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 11) + s := make([]string, 0, 12) s = append(s, "&api.ClusterSpec{") s = append(s, "Annotations: "+strings.Replace(this.Annotations.GoString(), `&`, ``, 1)+",\n") s = append(s, "AcceptancePolicy: "+strings.Replace(this.AcceptancePolicy.GoString(), `&`, ``, 1)+",\n") @@ -1168,6 +1171,7 @@ func (this *ClusterSpec) GoString() string { s = append(s, "Dispatcher: "+strings.Replace(this.Dispatcher.GoString(), `&`, ``, 1)+",\n") s = append(s, "CAConfig: "+strings.Replace(this.CAConfig.GoString(), `&`, ``, 1)+",\n") s = append(s, "TaskDefaults: "+strings.Replace(this.TaskDefaults.GoString(), `&`, ``, 1)+",\n") + s = append(s, "EncryptionConfig: "+strings.Replace(this.EncryptionConfig.GoString(), `&`, ``, 1)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -2008,6 +2012,14 @@ func (m *ClusterSpec) MarshalTo(data []byte) (int, error) { return 0, err } i += n29 + data[i] = 0x42 + i++ + i = encodeVarintSpecs(data, i, uint64(m.EncryptionConfig.Size())) + n30, err := m.EncryptionConfig.MarshalTo(data[i:]) + if err != nil { + return 0, err + } + i += n30 return i, nil } @@ -2029,11 +2041,11 @@ func (m *SecretSpec) MarshalTo(data []byte) (int, error) { data[i] = 0xa i++ i = encodeVarintSpecs(data, i, uint64(m.Annotations.Size())) - n30, err := m.Annotations.MarshalTo(data[i:]) + n31, err := m.Annotations.MarshalTo(data[i:]) if err != nil { return 0, err } - i += n30 + i += n31 if len(m.Data) > 0 { data[i] = 0x12 i++ @@ -2392,6 +2404,8 @@ func (m *ClusterSpec) Size() (n int) { n += 1 + l + sovSpecs(uint64(l)) l = m.TaskDefaults.Size() n += 1 + l + sovSpecs(uint64(l)) + l = m.EncryptionConfig.Size() + n += 1 + l + sovSpecs(uint64(l)) return n } @@ -2629,6 +2643,7 @@ func (this *ClusterSpec) String() string { `Dispatcher:` + strings.Replace(strings.Replace(this.Dispatcher.String(), "DispatcherConfig", "DispatcherConfig", 1), `&`, ``, 1) + `,`, `CAConfig:` + strings.Replace(strings.Replace(this.CAConfig.String(), "CAConfig", "CAConfig", 1), `&`, ``, 1) + `,`, `TaskDefaults:` + strings.Replace(strings.Replace(this.TaskDefaults.String(), "TaskDefaults", "TaskDefaults", 1), `&`, ``, 1) + `,`, + `EncryptionConfig:` + strings.Replace(strings.Replace(this.EncryptionConfig.String(), "EncryptionConfig", "EncryptionConfig", 1), `&`, ``, 1) + `,`, `}`, }, "") return s @@ -4956,6 +4971,36 @@ func (m *ClusterSpec) Unmarshal(data []byte) error { return err } iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EncryptionConfig", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowSpecs + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthSpecs + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.EncryptionConfig.Unmarshal(data[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipSpecs(data[iNdEx:]) @@ -5196,105 +5241,107 @@ var ( func init() { proto.RegisterFile("specs.proto", fileDescriptorSpecs) } var fileDescriptorSpecs = []byte{ - // 1597 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6e, 0xe3, 0xc8, - 0x11, 0x16, 0x2d, 0x59, 0x3f, 0x45, 0x69, 0x46, 0xd3, 0xd8, 0x1f, 0x8e, 0x76, 0x23, 0x69, 0xb4, - 0x93, 0x8d, 0x37, 0x8b, 0x78, 0x12, 0x25, 0xd8, 0xcc, 0x66, 0xb2, 0x48, 0xf4, 0x17, 0x8f, 0xe2, - 0xd8, 0x2b, 0xb4, 0xbd, 0x03, 0xcc, 0x49, 0x68, 0x93, 0x6d, 0x89, 0x30, 0xc5, 0x66, 0x9a, 0x4d, - 0x2d, 0x7c, 0xcb, 0x71, 0x31, 0x87, 0xbc, 0x81, 0x4f, 0x01, 0xf2, 0x06, 0xb9, 0xe4, 0x09, 0xe6, - 0x98, 0x63, 0x4e, 0x46, 0xac, 0x27, 0x08, 0x90, 0x17, 0x08, 0xba, 0xd9, 0x94, 0xa8, 0x2c, 0xbd, - 0x5e, 0x20, 0xbe, 0x75, 0x17, 0xbf, 0xaf, 0xd8, 0x5d, 0xf5, 0xb1, 0xaa, 0x08, 0x66, 0x18, 0x50, - 0x3b, 0xdc, 0x0f, 0x38, 0x13, 0x0c, 0x21, 0x87, 0xd9, 0x17, 0x94, 0xef, 0x87, 0x5f, 0x13, 0xbe, - 0xb8, 0x70, 0xc5, 0xfe, 0xf2, 0x67, 0x0d, 0x53, 0x5c, 0x06, 0x54, 0x03, 0x1a, 0xef, 0xcc, 0xd8, - 0x8c, 0xa9, 0xe5, 0x33, 0xb9, 0xd2, 0xd6, 0xf7, 0x9d, 0x88, 0x13, 0xe1, 0x32, 0xff, 0x59, 0xb2, - 0x88, 0x1f, 0x74, 0xfe, 0x5c, 0x80, 0xf2, 0x31, 0x73, 0xe8, 0x49, 0x40, 0x6d, 0x74, 0x00, 0x26, - 0xf1, 0x7d, 0x26, 0x14, 0x20, 0xb4, 0x8c, 0xb6, 0xb1, 0x67, 0x76, 0x5b, 0xfb, 0xdf, 0x7e, 0xe5, - 0x7e, 0x6f, 0x03, 0xeb, 0x17, 0xde, 0x5e, 0xb7, 0x72, 0x38, 0xcd, 0x44, 0x3f, 0x85, 0x02, 0x67, - 0x1e, 0xb5, 0x76, 0xda, 0xc6, 0xde, 0x83, 0xee, 0x87, 0x59, 0x1e, 0xe4, 0x4b, 0x31, 0xf3, 0x28, - 0x56, 0x48, 0x74, 0x00, 0xb0, 0xa0, 0x8b, 0x33, 0xca, 0xc3, 0xb9, 0x1b, 0x58, 0x79, 0xc5, 0xfb, - 0xd1, 0x6d, 0x3c, 0x79, 0xd8, 0xfd, 0xa3, 0x35, 0x1c, 0xa7, 0xa8, 0xe8, 0x08, 0xaa, 0x64, 0x49, - 0x5c, 0x8f, 0x9c, 0xb9, 0x9e, 0x2b, 0x2e, 0xad, 0x82, 0x72, 0xf5, 0xc9, 0x77, 0xba, 0xea, 0xa5, - 0x08, 0x78, 0x8b, 0xde, 0x71, 0x00, 0x36, 0x2f, 0x42, 0x1f, 0x43, 0x69, 0x32, 0x3a, 0x1e, 0x8e, - 0x8f, 0x0f, 0xea, 0xb9, 0xc6, 0xe3, 0x37, 0x57, 0xed, 0x77, 0xa5, 0x8f, 0x0d, 0x60, 0x42, 0x7d, - 0xc7, 0xf5, 0x67, 0x68, 0x0f, 0xca, 0xbd, 0xc1, 0x60, 0x34, 0x39, 0x1d, 0x0d, 0xeb, 0x46, 0xa3, - 0xf1, 0xe6, 0xaa, 0xfd, 0xde, 0x36, 0xb0, 0x67, 0xdb, 0x34, 0x10, 0xd4, 0x69, 0x14, 0xbe, 0xf9, - 0x4b, 0x33, 0xd7, 0xf9, 0xc6, 0x80, 0x6a, 0xfa, 0x10, 0xe8, 0x63, 0x28, 0xf6, 0x06, 0xa7, 0xe3, - 0x57, 0xa3, 0x7a, 0x6e, 0x43, 0x4f, 0x23, 0x7a, 0xb6, 0x70, 0x97, 0x14, 0x3d, 0x85, 0xdd, 0x49, - 0xef, 0xab, 0x93, 0x51, 0xdd, 0xd8, 0x1c, 0x27, 0x0d, 0x9b, 0x90, 0x28, 0x54, 0xa8, 0x21, 0xee, - 0x8d, 0x8f, 0xeb, 0x3b, 0xd9, 0xa8, 0x21, 0x27, 0xae, 0xaf, 0x8f, 0x72, 0x93, 0x07, 0xf3, 0x84, - 0xf2, 0xa5, 0x6b, 0xdf, 0xb3, 0x26, 0x3e, 0x83, 0x82, 0x20, 0xe1, 0x85, 0xd2, 0x84, 0x99, 0xad, - 0x89, 0x53, 0x12, 0x5e, 0xc8, 0x97, 0x6a, 0xba, 0xc2, 0x4b, 0x65, 0x70, 0x1a, 0x78, 0xae, 0x4d, - 0x04, 0x75, 0x94, 0x32, 0xcc, 0xee, 0x0f, 0xb3, 0xd8, 0x78, 0x8d, 0xd2, 0xe7, 0x7f, 0x99, 0xc3, - 0x29, 0x2a, 0x7a, 0x01, 0xc5, 0x99, 0xc7, 0xce, 0x88, 0xa7, 0x34, 0x61, 0x76, 0x9f, 0x64, 0x39, - 0x39, 0x50, 0x88, 0x8d, 0x03, 0x4d, 0x41, 0xcf, 0xa1, 0x18, 0x05, 0x0e, 0x11, 0xd4, 0x2a, 0x2a, - 0x72, 0x3b, 0x8b, 0xfc, 0x95, 0x42, 0x0c, 0x98, 0x7f, 0xee, 0xce, 0xb0, 0xc6, 0xa3, 0x43, 0x28, - 0xfb, 0x54, 0x7c, 0xcd, 0xf8, 0x45, 0x68, 0x95, 0xda, 0xf9, 0x3d, 0xb3, 0xfb, 0x69, 0xa6, 0x18, - 0x63, 0x4c, 0x4f, 0x08, 0x62, 0xcf, 0x17, 0xd4, 0x17, 0xb1, 0x9b, 0xfe, 0x8e, 0x65, 0xe0, 0xb5, - 0x03, 0xf4, 0x6b, 0x28, 0x53, 0xdf, 0x09, 0x98, 0xeb, 0x0b, 0xab, 0x7c, 0xfb, 0x41, 0x46, 0x1a, - 0x23, 0x83, 0x89, 0xd7, 0x8c, 0x7e, 0x11, 0x0a, 0x0b, 0xe6, 0xd0, 0xce, 0x33, 0x78, 0xf4, 0xad, - 0x60, 0xa1, 0x06, 0x94, 0x75, 0xb0, 0xe2, 0x2c, 0x17, 0xf0, 0x7a, 0xdf, 0x79, 0x08, 0xb5, 0xad, - 0xc0, 0xa8, 0xb2, 0x91, 0x64, 0x0b, 0xf5, 0xa0, 0x62, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b, - 0x24, 0x33, 0xb6, 0x83, 0x04, 0x24, 0x59, 0x2f, 0x73, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x0a, 0xa7, - 0x21, 0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xbd, 0xec, 0x1c, 0xc7, 0x20, 0x4c, 0xff, 0x18, 0xb9, - 0x9c, 0xca, 0x38, 0x85, 0x78, 0x43, 0x45, 0x2f, 0xa0, 0xc4, 0x69, 0x28, 0x08, 0x17, 0xdf, 0x95, - 0x64, 0x1c, 0x43, 0x26, 0xcc, 0x73, 0xed, 0x4b, 0x9c, 0x30, 0xd0, 0x0b, 0xa8, 0x04, 0x1e, 0xb1, - 0x95, 0x57, 0x6b, 0x57, 0xd1, 0x7f, 0x90, 0x45, 0x9f, 0x24, 0x20, 0xbc, 0xc1, 0xa3, 0xcf, 0x01, - 0x3c, 0x36, 0x9b, 0x3a, 0xdc, 0x5d, 0x52, 0xae, 0x45, 0xd2, 0xc8, 0x62, 0x0f, 0x15, 0x02, 0x57, - 0x3c, 0x36, 0x8b, 0x97, 0xe8, 0xe0, 0xff, 0x52, 0x48, 0x4a, 0x1d, 0x87, 0x00, 0x64, 0xfd, 0x54, - 0xeb, 0xe3, 0x93, 0xef, 0xe5, 0x4a, 0x67, 0x24, 0x45, 0x47, 0x4f, 0xa0, 0x7a, 0xce, 0xb8, 0x4d, - 0xa7, 0x5a, 0xf7, 0x15, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0x2b, 0x50, 0xe2, 0x91, 0x2f, - 0xdc, 0x05, 0xed, 0x1c, 0xc2, 0xbb, 0x99, 0x4e, 0x51, 0x17, 0xaa, 0xeb, 0x34, 0x4f, 0x5d, 0x47, - 0xe9, 0xa3, 0xd2, 0x7f, 0xb8, 0xba, 0x6e, 0x99, 0x6b, 0x3d, 0x8c, 0x87, 0xd8, 0x5c, 0x83, 0xc6, - 0x4e, 0xe7, 0xef, 0x25, 0xa8, 0x6d, 0x89, 0x05, 0xbd, 0x03, 0xbb, 0xee, 0x82, 0xcc, 0x68, 0x4c, - 0xc7, 0xf1, 0x06, 0x8d, 0xa0, 0xe8, 0x91, 0x33, 0xea, 0x49, 0xc9, 0xc8, 0xb0, 0xfd, 0xe4, 0x4e, - 0xd5, 0xed, 0xff, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x97, 0x58, 0x93, 0x91, 0x05, 0x25, 0x9b, 0x2d, - 0x16, 0xc4, 0x97, 0xe5, 0x25, 0xbf, 0x57, 0xc1, 0xc9, 0x16, 0x21, 0x28, 0x10, 0x3e, 0x0b, 0xad, - 0x82, 0x32, 0xab, 0x35, 0xaa, 0x43, 0x9e, 0xfa, 0x4b, 0x6b, 0x57, 0x99, 0xe4, 0x52, 0x5a, 0x1c, - 0x37, 0xce, 0x79, 0x05, 0xcb, 0xa5, 0xe4, 0x45, 0x21, 0xe5, 0x56, 0x49, 0x99, 0xd4, 0x1a, 0xfd, - 0x12, 0x8a, 0x0b, 0x16, 0xf9, 0x22, 0xb4, 0xca, 0xea, 0xb0, 0x8f, 0xb3, 0x0e, 0x7b, 0x24, 0x11, - 0xba, 0xfc, 0x69, 0x38, 0x7a, 0x09, 0x8f, 0x42, 0xc1, 0x82, 0xe9, 0x8c, 0x13, 0x9b, 0x4e, 0x03, - 0xca, 0x5d, 0xe6, 0xa8, 0x6c, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0x87, 0x92, 0x76, 0x20, - 0x59, 0x13, 0x45, 0x42, 0x13, 0xa8, 0x06, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e, - 0xbe, 0x47, 0xd4, 0x26, 0x91, 0xe7, 0x7d, 0x19, 0x93, 0xb0, 0x19, 0x6c, 0x36, 0xe8, 0x3d, 0x28, - 0xce, 0x38, 0x8b, 0x82, 0xd0, 0x32, 0x55, 0x3c, 0xf4, 0x0e, 0x7d, 0x01, 0xa5, 0x90, 0xda, 0x9c, - 0x8a, 0xd0, 0xaa, 0xaa, 0xdb, 0x7e, 0x94, 0xf5, 0x92, 0x13, 0x05, 0xc1, 0xf4, 0x9c, 0x72, 0xea, - 0xdb, 0x14, 0x27, 0x1c, 0xf4, 0x18, 0xf2, 0x42, 0x5c, 0x5a, 0xb5, 0xb6, 0xb1, 0x57, 0xee, 0x97, - 0x56, 0xd7, 0xad, 0xfc, 0xe9, 0xe9, 0x6b, 0x2c, 0x6d, 0xb2, 0x4c, 0xcd, 0x59, 0x28, 0x7c, 0xb2, - 0xa0, 0xd6, 0x03, 0x15, 0xde, 0xf5, 0x1e, 0xbd, 0x06, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2, - 0x7a, 0xa8, 0x6e, 0xf7, 0xe9, 0xdd, 0xb7, 0x1b, 0x1e, 0x9f, 0xe8, 0x62, 0x5b, 0x5b, 0x5d, 0xb7, - 0x2a, 0xeb, 0x2d, 0xae, 0x38, 0x7e, 0x18, 0x2f, 0x51, 0x1f, 0xcc, 0x39, 0x25, 0x9e, 0x98, 0xdb, - 0x73, 0x6a, 0x5f, 0x58, 0xf5, 0xdb, 0x6b, 0xef, 0x4b, 0x05, 0xd3, 0x1e, 0xd2, 0x24, 0x29, 0x62, - 0x79, 0xd4, 0xd0, 0x7a, 0xa4, 0x62, 0x15, 0x6f, 0x1a, 0x9f, 0x83, 0x99, 0x12, 0xa5, 0x14, 0xd3, - 0x05, 0xbd, 0xd4, 0x3a, 0x97, 0x4b, 0x49, 0x5b, 0x12, 0x2f, 0x8a, 0xa7, 0xa9, 0x0a, 0x8e, 0x37, - 0xbf, 0xda, 0x79, 0x6e, 0x34, 0xba, 0x60, 0xa6, 0x32, 0x83, 0x3e, 0x82, 0x1a, 0xa7, 0x33, 0x37, - 0x14, 0xfc, 0x72, 0x4a, 0x22, 0x31, 0xb7, 0x7e, 0xab, 0x08, 0xd5, 0xc4, 0xd8, 0x8b, 0xc4, 0xbc, - 0x31, 0x85, 0xcd, 0x05, 0x51, 0x1b, 0x4c, 0x19, 0xb8, 0x90, 0xf2, 0x25, 0xe5, 0xb2, 0xec, 0xcb, - 0x73, 0xa5, 0x4d, 0x32, 0xc1, 0x21, 0x25, 0xdc, 0x9e, 0xab, 0x4f, 0xac, 0x82, 0xf5, 0x4e, 0x7e, - 0x33, 0x89, 0x8a, 0xf4, 0x37, 0xa3, 0xb7, 0x9d, 0xff, 0x18, 0x50, 0x4d, 0xf7, 0x1f, 0x34, 0x88, - 0xbb, 0x8e, 0xba, 0xd2, 0x83, 0xee, 0xb3, 0xbb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x1d, - 0xc9, 0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x37, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb3, 0x2e, - 0x33, 0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x83, 0x6d, 0x6f, 0xe8, 0x29, 0xe4, 0x5f, 0x8d, - 0x27, 0xf5, 0x5c, 0xe3, 0x83, 0x37, 0x57, 0xed, 0xf7, 0xb7, 0x1f, 0xbe, 0x72, 0xb9, 0x88, 0x88, - 0x37, 0x9e, 0xa0, 0x1f, 0xc3, 0xee, 0xf0, 0xf8, 0x04, 0xe3, 0xba, 0xd1, 0x68, 0xbd, 0xb9, 0x6a, - 0x7f, 0xb0, 0x8d, 0x93, 0x8f, 0x58, 0xe4, 0x3b, 0x98, 0x9d, 0xad, 0xc7, 0xa6, 0xbf, 0xed, 0x80, - 0xa9, 0x0b, 0xe0, 0xfd, 0x8e, 0x4d, 0xbf, 0x81, 0x5a, 0xdc, 0x53, 0x12, 0x59, 0xef, 0xdc, 0xd9, - 0x5a, 0xaa, 0x31, 0x41, 0xe7, 0xf8, 0x09, 0x54, 0xdd, 0x60, 0xf9, 0xd9, 0x94, 0xfa, 0xe4, 0xcc, - 0xd3, 0x13, 0x54, 0x19, 0x9b, 0xd2, 0x36, 0x8a, 0x4d, 0xf2, 0x9b, 0x72, 0x7d, 0x41, 0xb9, 0xaf, - 0x67, 0xa3, 0x32, 0x5e, 0xef, 0xd1, 0x17, 0x50, 0x70, 0x03, 0xb2, 0xd0, 0xfd, 0x30, 0xf3, 0x06, - 0xe3, 0x49, 0xef, 0x48, 0x6b, 0xb0, 0x5f, 0x5e, 0x5d, 0xb7, 0x0a, 0xd2, 0x80, 0x15, 0x0d, 0x35, - 0x93, 0x96, 0x24, 0xdf, 0xa4, 0x4a, 0x64, 0x19, 0xa7, 0x2c, 0x9d, 0xbf, 0x16, 0xc0, 0x1c, 0x78, - 0x51, 0x28, 0x74, 0xa1, 0xbf, 0xb7, 0xb8, 0xbd, 0x86, 0x47, 0x44, 0x0d, 0xd9, 0xc4, 0x97, 0x55, - 0x53, 0xb5, 0x7a, 0x1d, 0xbb, 0xa7, 0x99, 0xee, 0xd6, 0xe0, 0x78, 0x2c, 0xe8, 0x17, 0xa5, 0x4f, - 0xcb, 0xc0, 0x75, 0xf2, 0x3f, 0x4f, 0xd0, 0x09, 0xd4, 0x18, 0xb7, 0xe7, 0x34, 0x14, 0x71, 0xa1, - 0xd5, 0x43, 0x69, 0xe6, 0xef, 0xca, 0x97, 0x69, 0xa0, 0xae, 0x32, 0xf1, 0x69, 0xb7, 0x7d, 0xa0, - 0xe7, 0x50, 0xe0, 0xe4, 0x3c, 0x19, 0x5b, 0x32, 0xf5, 0x8d, 0xc9, 0xb9, 0xd8, 0x72, 0xa1, 0x18, - 0xe8, 0xf7, 0x00, 0x8e, 0x1b, 0x06, 0x44, 0xd8, 0x73, 0xca, 0x75, 0x9e, 0x32, 0xaf, 0x38, 0x5c, - 0xa3, 0xb6, 0xbc, 0xa4, 0xd8, 0xe8, 0x10, 0x2a, 0x36, 0x49, 0x94, 0x56, 0xbc, 0xbd, 0xc7, 0x0c, - 0x7a, 0xda, 0x45, 0x5d, 0xba, 0x58, 0x5d, 0xb7, 0xca, 0x89, 0x05, 0x97, 0x6d, 0xa2, 0x95, 0x77, - 0x08, 0x35, 0x39, 0xc1, 0x4f, 0x1d, 0x7a, 0x4e, 0x22, 0x4f, 0x84, 0xaa, 0x1d, 0xde, 0x52, 0x35, - 0xe5, 0x30, 0x39, 0xd4, 0x38, 0x7d, 0xae, 0xaa, 0x48, 0xd9, 0x3a, 0x2e, 0x40, 0xdc, 0x2e, 0xee, - 0x57, 0x26, 0x08, 0x0a, 0x0e, 0x11, 0x44, 0x29, 0xa3, 0x8a, 0xd5, 0xba, 0xff, 0xe1, 0xdb, 0x9b, - 0x66, 0xee, 0x9f, 0x37, 0xcd, 0xdc, 0xbf, 0x6f, 0x9a, 0xc6, 0x9f, 0x56, 0x4d, 0xe3, 0xed, 0xaa, - 0x69, 0xfc, 0x63, 0xd5, 0x34, 0xfe, 0xb5, 0x6a, 0x1a, 0x67, 0x45, 0xf5, 0xe3, 0xfc, 0xf3, 0xff, - 0x06, 0x00, 0x00, 0xff, 0xff, 0x04, 0xd4, 0x09, 0xa4, 0x97, 0x0f, 0x00, 0x00, + // 1620 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x72, 0xdb, 0xc8, + 0x11, 0x26, 0x24, 0x8a, 0x3f, 0x0d, 0xca, 0xa6, 0xa6, 0xf6, 0x07, 0xe6, 0x6e, 0x48, 0x9a, 0xeb, + 0x6c, 0xb4, 0xd9, 0x8a, 0x9c, 0x30, 0xa9, 0x8d, 0x37, 0xce, 0x56, 0xc2, 0xbf, 0xc8, 0x8c, 0x22, + 0x2d, 0x6b, 0xa4, 0x75, 0xca, 0x27, 0xd6, 0x08, 0x18, 0x91, 0x28, 0x81, 0x18, 0x64, 0x30, 0xe0, + 0x16, 0x6f, 0x39, 0x6e, 0xf9, 0x90, 0x37, 0xf0, 0x29, 0xcf, 0x90, 0x4b, 0x9e, 0xc0, 0xc7, 0x1c, + 0x73, 0x52, 0xc5, 0x7c, 0x82, 0x54, 0xe5, 0x01, 0x92, 0x9a, 0xc1, 0x00, 0x04, 0x77, 0xa1, 0xb5, + 0xab, 0xe2, 0xdb, 0x4c, 0xe3, 0xfb, 0x1a, 0x3d, 0x3d, 0x1f, 0xba, 0x1b, 0x60, 0x86, 0x01, 0xb5, + 0xc3, 0xa3, 0x80, 0x33, 0xc1, 0x10, 0x72, 0x98, 0x7d, 0x4d, 0xf9, 0x51, 0xf8, 0x35, 0xe1, 0x8b, + 0x6b, 0x57, 0x1c, 0x2d, 0x7f, 0xd6, 0x30, 0xc5, 0x2a, 0xa0, 0x1a, 0xd0, 0x78, 0x67, 0xc6, 0x66, + 0x4c, 0x2d, 0x1f, 0xca, 0x95, 0xb6, 0xbe, 0xef, 0x44, 0x9c, 0x08, 0x97, 0xf9, 0x0f, 0x93, 0x45, + 0xfc, 0xa0, 0xf3, 0x97, 0x22, 0x54, 0xce, 0x98, 0x43, 0xcf, 0x03, 0x6a, 0xa3, 0x63, 0x30, 0x89, + 0xef, 0x33, 0xa1, 0x00, 0xa1, 0x65, 0xb4, 0x8d, 0x43, 0xb3, 0xdb, 0x3a, 0xfa, 0xee, 0x2b, 0x8f, + 0x7a, 0x1b, 0x58, 0xbf, 0xf8, 0xf2, 0xa6, 0x55, 0xc0, 0x59, 0x26, 0xfa, 0x29, 0x14, 0x39, 0xf3, + 0xa8, 0xb5, 0xd3, 0x36, 0x0e, 0xef, 0x74, 0x3f, 0xcc, 0xf3, 0x20, 0x5f, 0x8a, 0x99, 0x47, 0xb1, + 0x42, 0xa2, 0x63, 0x80, 0x05, 0x5d, 0x5c, 0x52, 0x1e, 0xce, 0xdd, 0xc0, 0xda, 0x55, 0xbc, 0x1f, + 0xdd, 0xc6, 0x93, 0xc1, 0x1e, 0x9d, 0xa6, 0x70, 0x9c, 0xa1, 0xa2, 0x53, 0xa8, 0x91, 0x25, 0x71, + 0x3d, 0x72, 0xe9, 0x7a, 0xae, 0x58, 0x59, 0x45, 0xe5, 0xea, 0x93, 0xef, 0x75, 0xd5, 0xcb, 0x10, + 0xf0, 0x16, 0xbd, 0xe3, 0x00, 0x6c, 0x5e, 0x84, 0x3e, 0x86, 0xf2, 0x64, 0x74, 0x36, 0x1c, 0x9f, + 0x1d, 0xd7, 0x0b, 0x8d, 0x7b, 0xcf, 0x5f, 0xb4, 0xdf, 0x95, 0x3e, 0x36, 0x80, 0x09, 0xf5, 0x1d, + 0xd7, 0x9f, 0xa1, 0x43, 0xa8, 0xf4, 0x06, 0x83, 0xd1, 0xe4, 0x62, 0x34, 0xac, 0x1b, 0x8d, 0xc6, + 0xf3, 0x17, 0xed, 0xf7, 0xb6, 0x81, 0x3d, 0xdb, 0xa6, 0x81, 0xa0, 0x4e, 0xa3, 0xf8, 0xcd, 0x5f, + 0x9b, 0x85, 0xce, 0x37, 0x06, 0xd4, 0xb2, 0x41, 0xa0, 0x8f, 0xa1, 0xd4, 0x1b, 0x5c, 0x8c, 0x9f, + 0x8e, 0xea, 0x85, 0x0d, 0x3d, 0x8b, 0xe8, 0xd9, 0xc2, 0x5d, 0x52, 0xf4, 0x00, 0xf6, 0x26, 0xbd, + 0xaf, 0xce, 0x47, 0x75, 0x63, 0x13, 0x4e, 0x16, 0x36, 0x21, 0x51, 0xa8, 0x50, 0x43, 0xdc, 0x1b, + 0x9f, 0xd5, 0x77, 0xf2, 0x51, 0x43, 0x4e, 0x5c, 0x5f, 0x87, 0xf2, 0x6a, 0x17, 0xcc, 0x73, 0xca, + 0x97, 0xae, 0xfd, 0x96, 0x35, 0xf1, 0x19, 0x14, 0x05, 0x09, 0xaf, 0x95, 0x26, 0xcc, 0x7c, 0x4d, + 0x5c, 0x90, 0xf0, 0x5a, 0xbe, 0x54, 0xd3, 0x15, 0x5e, 0x2a, 0x83, 0xd3, 0xc0, 0x73, 0x6d, 0x22, + 0xa8, 0xa3, 0x94, 0x61, 0x76, 0x7f, 0x98, 0xc7, 0xc6, 0x29, 0x4a, 0xc7, 0xff, 0xa4, 0x80, 0x33, + 0x54, 0xf4, 0x18, 0x4a, 0x33, 0x8f, 0x5d, 0x12, 0x4f, 0x69, 0xc2, 0xec, 0xde, 0xcf, 0x73, 0x72, + 0xac, 0x10, 0x1b, 0x07, 0x9a, 0x82, 0x1e, 0x41, 0x29, 0x0a, 0x1c, 0x22, 0xa8, 0x55, 0x52, 0xe4, + 0x76, 0x1e, 0xf9, 0x2b, 0x85, 0x18, 0x30, 0xff, 0xca, 0x9d, 0x61, 0x8d, 0x47, 0x27, 0x50, 0xf1, + 0xa9, 0xf8, 0x9a, 0xf1, 0xeb, 0xd0, 0x2a, 0xb7, 0x77, 0x0f, 0xcd, 0xee, 0xa7, 0xb9, 0x62, 0x8c, + 0x31, 0x3d, 0x21, 0x88, 0x3d, 0x5f, 0x50, 0x5f, 0xc4, 0x6e, 0xfa, 0x3b, 0x96, 0x81, 0x53, 0x07, + 0xe8, 0xd7, 0x50, 0xa1, 0xbe, 0x13, 0x30, 0xd7, 0x17, 0x56, 0xe5, 0xf6, 0x40, 0x46, 0x1a, 0x23, + 0x93, 0x89, 0x53, 0x46, 0xbf, 0x04, 0xc5, 0x05, 0x73, 0x68, 0xe7, 0x21, 0x1c, 0x7c, 0x27, 0x59, + 0xa8, 0x01, 0x15, 0x9d, 0xac, 0xf8, 0x96, 0x8b, 0x38, 0xdd, 0x77, 0xee, 0xc2, 0xfe, 0x56, 0x62, + 0x54, 0xd9, 0x48, 0x6e, 0x0b, 0xf5, 0xa0, 0x6a, 0x33, 0x5f, 0x10, 0xd7, 0xa7, 0x5c, 0x0b, 0x24, + 0x37, 0xb7, 0x83, 0x04, 0x24, 0x59, 0x4f, 0x0a, 0x78, 0xc3, 0x42, 0xbf, 0x83, 0x2a, 0xa7, 0x21, + 0x8b, 0xb8, 0x4d, 0x43, 0xad, 0x90, 0xc3, 0xfc, 0x3b, 0x8e, 0x41, 0x98, 0xfe, 0x29, 0x72, 0x39, + 0x95, 0x79, 0x0a, 0xf1, 0x86, 0x8a, 0x1e, 0x43, 0x99, 0xd3, 0x50, 0x10, 0x2e, 0xbe, 0xef, 0x92, + 0x71, 0x0c, 0x99, 0x30, 0xcf, 0xb5, 0x57, 0x38, 0x61, 0xa0, 0xc7, 0x50, 0x0d, 0x3c, 0x62, 0x2b, + 0xaf, 0xd6, 0x9e, 0xa2, 0xff, 0x20, 0x8f, 0x3e, 0x49, 0x40, 0x78, 0x83, 0x47, 0x9f, 0x03, 0x78, + 0x6c, 0x36, 0x75, 0xb8, 0xbb, 0xa4, 0x5c, 0x8b, 0xa4, 0x91, 0xc7, 0x1e, 0x2a, 0x04, 0xae, 0x7a, + 0x6c, 0x16, 0x2f, 0xd1, 0xf1, 0xff, 0xa5, 0x90, 0x8c, 0x3a, 0x4e, 0x00, 0x48, 0xfa, 0x54, 0xeb, + 0xe3, 0x93, 0x37, 0x72, 0xa5, 0x6f, 0x24, 0x43, 0x47, 0xf7, 0xa1, 0x76, 0xc5, 0xb8, 0x4d, 0xa7, + 0x5a, 0xf7, 0x55, 0xa5, 0x09, 0x53, 0xd9, 0x62, 0xa1, 0xf7, 0xab, 0x50, 0xe6, 0x91, 0x2f, 0xdc, + 0x05, 0xed, 0x9c, 0xc0, 0xbb, 0xb9, 0x4e, 0x51, 0x17, 0x6a, 0xe9, 0x35, 0x4f, 0x5d, 0x47, 0xe9, + 0xa3, 0xda, 0xbf, 0xbb, 0xbe, 0x69, 0x99, 0xa9, 0x1e, 0xc6, 0x43, 0x6c, 0xa6, 0xa0, 0xb1, 0xd3, + 0xf9, 0x7b, 0x19, 0xf6, 0xb7, 0xc4, 0x82, 0xde, 0x81, 0x3d, 0x77, 0x41, 0x66, 0x34, 0xa6, 0xe3, + 0x78, 0x83, 0x46, 0x50, 0xf2, 0xc8, 0x25, 0xf5, 0xa4, 0x64, 0x64, 0xda, 0x7e, 0xf2, 0x5a, 0xd5, + 0x1d, 0xfd, 0x41, 0xe1, 0x47, 0xbe, 0xe0, 0x2b, 0xac, 0xc9, 0xc8, 0x82, 0xb2, 0xcd, 0x16, 0x0b, + 0xe2, 0xcb, 0xf2, 0xb2, 0x7b, 0x58, 0xc5, 0xc9, 0x16, 0x21, 0x28, 0x12, 0x3e, 0x0b, 0xad, 0xa2, + 0x32, 0xab, 0x35, 0xaa, 0xc3, 0x2e, 0xf5, 0x97, 0xd6, 0x9e, 0x32, 0xc9, 0xa5, 0xb4, 0x38, 0x6e, + 0x7c, 0xe7, 0x55, 0x2c, 0x97, 0x92, 0x17, 0x85, 0x94, 0x5b, 0x65, 0x65, 0x52, 0x6b, 0xf4, 0x4b, + 0x28, 0x2d, 0x58, 0xe4, 0x8b, 0xd0, 0xaa, 0xa8, 0x60, 0xef, 0xe5, 0x05, 0x7b, 0x2a, 0x11, 0xba, + 0xfc, 0x69, 0x38, 0x7a, 0x02, 0x07, 0xa1, 0x60, 0xc1, 0x74, 0xc6, 0x89, 0x4d, 0xa7, 0x01, 0xe5, + 0x2e, 0x73, 0xd4, 0x6d, 0xdc, 0x52, 0x45, 0x87, 0xba, 0xc3, 0xe3, 0xbb, 0x92, 0x76, 0x2c, 0x59, + 0x13, 0x45, 0x42, 0x13, 0xa8, 0x05, 0x91, 0xe7, 0x4d, 0x59, 0x10, 0x17, 0x73, 0x50, 0x4e, 0xde, + 0x20, 0x6b, 0x93, 0xc8, 0xf3, 0xbe, 0x8c, 0x49, 0xd8, 0x0c, 0x36, 0x1b, 0xf4, 0x1e, 0x94, 0x66, + 0x9c, 0x45, 0x41, 0x68, 0x99, 0x2a, 0x1f, 0x7a, 0x87, 0xbe, 0x80, 0x72, 0x48, 0x6d, 0x4e, 0x45, + 0x68, 0xd5, 0xd4, 0x69, 0x3f, 0xca, 0x7b, 0xc9, 0xb9, 0x82, 0x60, 0x7a, 0x45, 0x39, 0xf5, 0x6d, + 0x8a, 0x13, 0x0e, 0xba, 0x07, 0xbb, 0x42, 0xac, 0xac, 0xfd, 0xb6, 0x71, 0x58, 0xe9, 0x97, 0xd7, + 0x37, 0xad, 0xdd, 0x8b, 0x8b, 0x67, 0x58, 0xda, 0x64, 0x99, 0x9a, 0xb3, 0x50, 0xf8, 0x64, 0x41, + 0xad, 0x3b, 0x2a, 0xbd, 0xe9, 0x1e, 0x3d, 0x03, 0x70, 0xfc, 0x70, 0x6a, 0xab, 0xef, 0xc2, 0xba, + 0xab, 0x4e, 0xf7, 0xe9, 0xeb, 0x4f, 0x37, 0x3c, 0x3b, 0xd7, 0xc5, 0x76, 0x7f, 0x7d, 0xd3, 0xaa, + 0xa6, 0x5b, 0x5c, 0x75, 0xfc, 0x30, 0x5e, 0xa2, 0x3e, 0x98, 0x73, 0x4a, 0x3c, 0x31, 0xb7, 0xe7, + 0xd4, 0xbe, 0xb6, 0xea, 0xb7, 0xd7, 0xde, 0x27, 0x0a, 0xa6, 0x3d, 0x64, 0x49, 0x52, 0xc4, 0x32, + 0xd4, 0xd0, 0x3a, 0x50, 0xb9, 0x8a, 0x37, 0x8d, 0xcf, 0xc1, 0xcc, 0x88, 0x52, 0x8a, 0xe9, 0x9a, + 0xae, 0xb4, 0xce, 0xe5, 0x52, 0xd2, 0x96, 0xc4, 0x8b, 0xe2, 0x69, 0xaa, 0x8a, 0xe3, 0xcd, 0xaf, + 0x76, 0x1e, 0x19, 0x8d, 0x2e, 0x98, 0x99, 0x9b, 0x41, 0x1f, 0xc1, 0x3e, 0xa7, 0x33, 0x37, 0x14, + 0x7c, 0x35, 0x25, 0x91, 0x98, 0x5b, 0xbf, 0x55, 0x84, 0x5a, 0x62, 0xec, 0x45, 0x62, 0xde, 0x98, + 0xc2, 0xe6, 0x80, 0xa8, 0x0d, 0xa6, 0x4c, 0x5c, 0x48, 0xf9, 0x92, 0x72, 0x59, 0xf6, 0x65, 0x5c, + 0x59, 0x93, 0xbc, 0xe0, 0x90, 0x12, 0x6e, 0xcf, 0xd5, 0x27, 0x56, 0xc5, 0x7a, 0x27, 0xbf, 0x99, + 0x44, 0x45, 0xfa, 0x9b, 0xd1, 0xdb, 0xce, 0x7f, 0x0c, 0xa8, 0x65, 0xfb, 0x0f, 0x1a, 0xc4, 0x5d, + 0x47, 0x1d, 0xe9, 0x4e, 0xf7, 0xe1, 0xeb, 0xfa, 0x95, 0xaa, 0xf1, 0x5e, 0x24, 0x9d, 0x9d, 0xca, + 0x19, 0x51, 0x91, 0xd1, 0x2f, 0x60, 0x2f, 0x60, 0x5c, 0x24, 0x5f, 0x7a, 0x33, 0xb7, 0x2e, 0x33, + 0x9e, 0xd4, 0xc4, 0x18, 0xdc, 0x99, 0xc3, 0x9d, 0x6d, 0x6f, 0xe8, 0x01, 0xec, 0x3e, 0x1d, 0x4f, + 0xea, 0x85, 0xc6, 0x07, 0xcf, 0x5f, 0xb4, 0xdf, 0xdf, 0x7e, 0xf8, 0xd4, 0xe5, 0x22, 0x22, 0xde, + 0x78, 0x82, 0x7e, 0x0c, 0x7b, 0xc3, 0xb3, 0x73, 0x8c, 0xeb, 0x46, 0xa3, 0xf5, 0xfc, 0x45, 0xfb, + 0x83, 0x6d, 0x9c, 0x7c, 0xc4, 0x22, 0xdf, 0xc1, 0xec, 0x32, 0x1d, 0x9b, 0xfe, 0xb6, 0x03, 0xa6, + 0x2e, 0x80, 0x6f, 0x77, 0x6c, 0xfa, 0x0d, 0xec, 0xc7, 0x3d, 0x25, 0x91, 0xf5, 0xce, 0x6b, 0x5b, + 0x4b, 0x2d, 0x26, 0xe8, 0x3b, 0xbe, 0x0f, 0x35, 0x37, 0x58, 0x7e, 0x36, 0xa5, 0x3e, 0xb9, 0xf4, + 0xf4, 0x04, 0x55, 0xc1, 0xa6, 0xb4, 0x8d, 0x62, 0x93, 0xfc, 0xa6, 0x5c, 0x5f, 0x50, 0xee, 0xeb, + 0xd9, 0xa8, 0x82, 0xd3, 0x3d, 0xfa, 0x02, 0x8a, 0x6e, 0x40, 0x16, 0xba, 0x1f, 0xe6, 0x9e, 0x60, + 0x3c, 0xe9, 0x9d, 0x6a, 0x0d, 0xf6, 0x2b, 0xeb, 0x9b, 0x56, 0x51, 0x1a, 0xb0, 0xa2, 0xa1, 0x66, + 0xd2, 0x92, 0xe4, 0x9b, 0x54, 0x89, 0xac, 0xe0, 0x8c, 0xa5, 0xf3, 0xdf, 0x22, 0x98, 0x03, 0x2f, + 0x0a, 0x85, 0x2e, 0xf4, 0x6f, 0x2d, 0x6f, 0xcf, 0xe0, 0x80, 0xa8, 0x21, 0x9b, 0xf8, 0xb2, 0x6a, + 0xaa, 0x56, 0xaf, 0x73, 0xf7, 0x20, 0xd7, 0x5d, 0x0a, 0x8e, 0xc7, 0x82, 0x7e, 0x49, 0xfa, 0xb4, + 0x0c, 0x5c, 0x27, 0xdf, 0x7a, 0x82, 0xce, 0x61, 0x9f, 0x71, 0x7b, 0x4e, 0x43, 0x11, 0x17, 0x5a, + 0x3d, 0x94, 0xe6, 0xfe, 0xae, 0x7c, 0x99, 0x05, 0xea, 0x2a, 0x13, 0x47, 0xbb, 0xed, 0x03, 0x3d, + 0x82, 0x22, 0x27, 0x57, 0xc9, 0xd8, 0x92, 0xab, 0x6f, 0x4c, 0xae, 0xc4, 0x96, 0x0b, 0xc5, 0x40, + 0xbf, 0x07, 0x70, 0xdc, 0x30, 0x20, 0xc2, 0x9e, 0x53, 0xae, 0xef, 0x29, 0xf7, 0x88, 0xc3, 0x14, + 0xb5, 0xe5, 0x25, 0xc3, 0x46, 0x27, 0x50, 0xb5, 0x49, 0xa2, 0xb4, 0xd2, 0xed, 0x3d, 0x66, 0xd0, + 0xd3, 0x2e, 0xea, 0xd2, 0xc5, 0xfa, 0xa6, 0x55, 0x49, 0x2c, 0xb8, 0x62, 0x13, 0xad, 0xbc, 0x13, + 0xd8, 0x97, 0x13, 0xfc, 0xd4, 0xa1, 0x57, 0x24, 0xf2, 0x44, 0xa8, 0xda, 0xe1, 0x2d, 0x55, 0x53, + 0x0e, 0x93, 0x43, 0x8d, 0xd3, 0x71, 0xd5, 0x44, 0xc6, 0x86, 0xfe, 0x08, 0x07, 0xd4, 0xb7, 0xf9, + 0x4a, 0xe9, 0x2c, 0x89, 0xb0, 0x72, 0xfb, 0x61, 0x47, 0x29, 0x78, 0xeb, 0xb0, 0x75, 0xfa, 0x2d, + 0x7b, 0xc7, 0x05, 0x88, 0xfb, 0xd0, 0xdb, 0xd5, 0x1f, 0x82, 0xa2, 0x43, 0x04, 0x51, 0x92, 0xab, + 0x61, 0xb5, 0xee, 0x7f, 0xf8, 0xf2, 0x55, 0xb3, 0xf0, 0xcf, 0x57, 0xcd, 0xc2, 0xbf, 0x5f, 0x35, + 0x8d, 0x3f, 0xaf, 0x9b, 0xc6, 0xcb, 0x75, 0xd3, 0xf8, 0xc7, 0xba, 0x69, 0xfc, 0x6b, 0xdd, 0x34, + 0x2e, 0x4b, 0xea, 0x8f, 0xfc, 0xe7, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xce, 0x13, 0x97, 0xa2, + 0xf0, 0x0f, 0x00, 0x00, } diff --git a/api/specs.proto b/api/specs.proto index 4e3d0dded4..ae2b7874a3 100644 --- a/api/specs.proto +++ b/api/specs.proto @@ -323,6 +323,9 @@ message ClusterSpec { // TaskDefaults specifies the default values to use for task creation. TaskDefaults task_defaults = 7 [(gogoproto.nullable) = false]; + + // EncryptionConfig defines the cluster's encryption settings. + EncryptionConfig encryption_config = 8 [(gogoproto.nullable) = false]; } // SecretSpec specifies a user-provided secret. diff --git a/api/types.pb.go b/api/types.pb.go index 7fbe601325..f1df343734 100644 --- a/api/types.pb.go +++ b/api/types.pb.go @@ -52,6 +52,7 @@ TaskDefaults DispatcherConfig RaftConfig + EncryptionConfig Placement JoinTokens RootCA @@ -118,7 +119,7 @@ GetClusterResponse ListClustersRequest ListClustersResponse - JoinTokenRotation + KeyRotation UpdateClusterRequest UpdateClusterResponse GetSecretRequest @@ -149,6 +150,8 @@ IssueNodeCertificateResponse GetRootCACertificateRequest GetRootCACertificateResponse + GetUnlockKeyRequest + GetUnlockKeyResponse StoreSnapshot ClusterSnapshot Snapshot @@ -662,7 +665,7 @@ func (x EncryptionKey_Algorithm) String() string { return proto.EnumName(EncryptionKey_Algorithm_name, int32(x)) } func (EncryptionKey_Algorithm) EnumDescriptor() ([]byte, []int) { - return fileDescriptorTypes, []int{37, 0} + return fileDescriptorTypes, []int{38, 0} } type MaybeEncryptedRecord_Algorithm int32 @@ -685,7 +688,7 @@ func (x MaybeEncryptedRecord_Algorithm) String() string { return proto.EnumName(MaybeEncryptedRecord_Algorithm_name, int32(x)) } func (MaybeEncryptedRecord_Algorithm) EnumDescriptor() ([]byte, []int) { - return fileDescriptorTypes, []int{42, 0} + return fileDescriptorTypes, []int{43, 0} } // Version tracks the last time an object in the store was updated. @@ -1359,6 +1362,17 @@ func (m *RaftConfig) Reset() { *m = RaftConfig{} } func (*RaftConfig) ProtoMessage() {} func (*RaftConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{32} } +type EncryptionConfig struct { + // AutoLockManagers specifies whether or not managers TLS keys and raft data + // should be encrypted at rest in such a way that they must be unlocked + // before the manager node starts up again. + AutoLockManagers bool `protobuf:"varint,1,opt,name=auto_lock_managers,json=autoLockManagers,proto3" json:"auto_lock_managers,omitempty"` +} + +func (m *EncryptionConfig) Reset() { *m = EncryptionConfig{} } +func (*EncryptionConfig) ProtoMessage() {} +func (*EncryptionConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} } + // Placement specifies task distribution constraints. type Placement struct { // constraints specifies a set of requirements a node should meet for a task. @@ -1367,7 +1381,7 @@ type Placement struct { func (m *Placement) Reset() { *m = Placement{} } func (*Placement) ProtoMessage() {} -func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{33} } +func (*Placement) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} } // JoinToken contains the join tokens for workers and managers. type JoinTokens struct { @@ -1379,7 +1393,7 @@ type JoinTokens struct { func (m *JoinTokens) Reset() { *m = JoinTokens{} } func (*JoinTokens) ProtoMessage() {} -func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{34} } +func (*JoinTokens) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} } type RootCA struct { // CAKey is the root CA private key. @@ -1394,7 +1408,7 @@ type RootCA struct { func (m *RootCA) Reset() { *m = RootCA{} } func (*RootCA) ProtoMessage() {} -func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{35} } +func (*RootCA) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} } type Certificate struct { Role NodeRole `protobuf:"varint,1,opt,name=role,proto3,enum=docker.swarmkit.v1.NodeRole" json:"role,omitempty"` @@ -1407,7 +1421,7 @@ type Certificate struct { func (m *Certificate) Reset() { *m = Certificate{} } func (*Certificate) ProtoMessage() {} -func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{36} } +func (*Certificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} } // Symmetric keys to encrypt inter-agent communication. type EncryptionKey struct { @@ -1423,7 +1437,7 @@ type EncryptionKey struct { func (m *EncryptionKey) Reset() { *m = EncryptionKey{} } func (*EncryptionKey) ProtoMessage() {} -func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{37} } +func (*EncryptionKey) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} } // ManagerStatus provides informations about the state of a manager in the cluster. type ManagerStatus struct { @@ -1440,7 +1454,7 @@ type ManagerStatus struct { func (m *ManagerStatus) Reset() { *m = ManagerStatus{} } func (*ManagerStatus) ProtoMessage() {} -func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{38} } +func (*ManagerStatus) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} } // SecretReference is the linkage between a service and a secret that it uses. type SecretReference struct { @@ -1460,7 +1474,7 @@ type SecretReference struct { func (m *SecretReference) Reset() { *m = SecretReference{} } func (*SecretReference) ProtoMessage() {} -func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{39} } +func (*SecretReference) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} } type isSecretReference_Target interface { isSecretReference_Target() @@ -1558,7 +1572,7 @@ type SecretReference_FileTarget struct { func (m *SecretReference_FileTarget) Reset() { *m = SecretReference_FileTarget{} } func (*SecretReference_FileTarget) ProtoMessage() {} func (*SecretReference_FileTarget) Descriptor() ([]byte, []int) { - return fileDescriptorTypes, []int{39, 0} + return fileDescriptorTypes, []int{40, 0} } // BlacklistedCertificate is a record for a blacklisted certificate. It does not @@ -1571,7 +1585,7 @@ type BlacklistedCertificate struct { func (m *BlacklistedCertificate) Reset() { *m = BlacklistedCertificate{} } func (*BlacklistedCertificate) ProtoMessage() {} -func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{40} } +func (*BlacklistedCertificate) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} } // HealthConfig holds configuration settings for the HEALTHCHECK feature. type HealthConfig struct { @@ -1595,7 +1609,7 @@ type HealthConfig struct { func (m *HealthConfig) Reset() { *m = HealthConfig{} } func (*HealthConfig) ProtoMessage() {} -func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{41} } +func (*HealthConfig) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} } type MaybeEncryptedRecord struct { Algorithm MaybeEncryptedRecord_Algorithm `protobuf:"varint,1,opt,name=algorithm,proto3,enum=docker.swarmkit.v1.MaybeEncryptedRecord_Algorithm" json:"algorithm,omitempty"` @@ -1605,7 +1619,7 @@ type MaybeEncryptedRecord struct { func (m *MaybeEncryptedRecord) Reset() { *m = MaybeEncryptedRecord{} } func (*MaybeEncryptedRecord) ProtoMessage() {} -func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{42} } +func (*MaybeEncryptedRecord) Descriptor() ([]byte, []int) { return fileDescriptorTypes, []int{43} } func init() { proto.RegisterType((*Version)(nil), "docker.swarmkit.v1.Version") @@ -1646,6 +1660,7 @@ func init() { proto.RegisterType((*TaskDefaults)(nil), "docker.swarmkit.v1.TaskDefaults") proto.RegisterType((*DispatcherConfig)(nil), "docker.swarmkit.v1.DispatcherConfig") proto.RegisterType((*RaftConfig)(nil), "docker.swarmkit.v1.RaftConfig") + proto.RegisterType((*EncryptionConfig)(nil), "docker.swarmkit.v1.EncryptionConfig") proto.RegisterType((*Placement)(nil), "docker.swarmkit.v1.Placement") proto.RegisterType((*JoinTokens)(nil), "docker.swarmkit.v1.JoinTokens") proto.RegisterType((*RootCA)(nil), "docker.swarmkit.v1.RootCA") @@ -2276,6 +2291,18 @@ func (m *RaftConfig) Copy() *RaftConfig { return o } +func (m *EncryptionConfig) Copy() *EncryptionConfig { + if m == nil { + return nil + } + + o := &EncryptionConfig{ + AutoLockManagers: m.AutoLockManagers, + } + + return o +} + func (m *Placement) Copy() *Placement { if m == nil { return nil @@ -3028,6 +3055,16 @@ func (this *RaftConfig) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *EncryptionConfig) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&api.EncryptionConfig{") + s = append(s, "AutoLockManagers: "+fmt.Sprintf("%#v", this.AutoLockManagers)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func (this *Placement) GoString() string { if this == nil { return "nil" @@ -4708,6 +4745,34 @@ func (m *RaftConfig) MarshalTo(data []byte) (int, error) { return i, nil } +func (m *EncryptionConfig) Marshal() (data []byte, err error) { + size := m.Size() + data = make([]byte, size) + n, err := m.MarshalTo(data) + if err != nil { + return nil, err + } + return data[:n], nil +} + +func (m *EncryptionConfig) MarshalTo(data []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.AutoLockManagers { + data[i] = 0x8 + i++ + if m.AutoLockManagers { + data[i] = 1 + } else { + data[i] = 0 + } + i++ + } + return i, nil +} + func (m *Placement) Marshal() (data []byte, err error) { size := m.Size() data = make([]byte, size) @@ -5836,6 +5901,15 @@ func (m *RaftConfig) Size() (n int) { return n } +func (m *EncryptionConfig) Size() (n int) { + var l int + _ = l + if m.AutoLockManagers { + n += 2 + } + return n +} + func (m *Placement) Size() (n int) { var l int _ = l @@ -6569,6 +6643,16 @@ func (this *RaftConfig) String() string { }, "") return s } +func (this *EncryptionConfig) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&EncryptionConfig{`, + `AutoLockManagers:` + fmt.Sprintf("%v", this.AutoLockManagers) + `,`, + `}`, + }, "") + return s +} func (this *Placement) String() string { if this == nil { return "nil" @@ -11863,6 +11947,76 @@ func (m *RaftConfig) Unmarshal(data []byte) error { } return nil } +func (m *EncryptionConfig) Unmarshal(data []byte) error { + l := len(data) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: EncryptionConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: EncryptionConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AutoLockManagers", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := data[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.AutoLockManagers = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipTypes(data[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Placement) Unmarshal(data []byte) error { l := len(data) iNdEx := 0 @@ -13478,252 +13632,254 @@ var ( func init() { proto.RegisterFile("types.proto", fileDescriptorTypes) } var fileDescriptorTypes = []byte{ - // 3946 bytes of a gzipped FileDescriptorProto + // 3975 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x79, 0x4d, 0x6c, 0x1b, 0x49, - 0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0xcf, - 0x78, 0x67, 0xc6, 0x3b, 0x7f, 0x8e, 0xad, 0xf9, 0x80, 0x77, 0xfc, 0xcf, 0x7a, 0x9a, 0x1f, 0xb2, - 0xb8, 0x96, 0x48, 0xa2, 0x48, 0xd9, 0x19, 0x04, 0x08, 0x51, 0xea, 0x2e, 0x91, 0x3d, 0x6a, 0x76, - 0x33, 0xdd, 0x45, 0xc9, 0x4c, 0x10, 0xc4, 0xc8, 0x21, 0x09, 0x74, 0xca, 0x3d, 0x10, 0x82, 0x20, - 0x41, 0x0e, 0x39, 0xec, 0x25, 0x87, 0x00, 0x39, 0x0d, 0x72, 0x9a, 0xe3, 0x26, 0x01, 0x82, 0x45, - 0x82, 0x18, 0x19, 0xe5, 0x1c, 0x60, 0x2f, 0x8b, 0x1c, 0x92, 0x00, 0x41, 0x7d, 0x74, 0xb3, 0x29, - 0xd3, 0xb2, 0x27, 0xbb, 0x17, 0xb2, 0xeb, 0xd5, 0xef, 0xbd, 0xfa, 0x7a, 0x55, 0xf5, 0x7b, 0xaf, - 0xa0, 0xc0, 0xa6, 0x63, 0x1a, 0x54, 0xc6, 0xbe, 0xc7, 0x3c, 0x84, 0x2c, 0xcf, 0x3c, 0xa2, 0x7e, - 0x25, 0x38, 0x21, 0xfe, 0xe8, 0xc8, 0x66, 0x95, 0xe3, 0xbb, 0xa5, 0x1b, 0xcc, 0x1e, 0xd1, 0x80, - 0x91, 0xd1, 0xf8, 0xa3, 0xe8, 0x4b, 0xc2, 0x4b, 0x6f, 0x58, 0x13, 0x9f, 0x30, 0xdb, 0x73, 0x3f, - 0x0a, 0x3f, 0x54, 0xc5, 0xb5, 0x81, 0x37, 0xf0, 0xc4, 0xe7, 0x47, 0xfc, 0x4b, 0x4a, 0xf5, 0x4d, - 0x58, 0x7e, 0x4c, 0xfd, 0xc0, 0xf6, 0x5c, 0x74, 0x0d, 0x32, 0xb6, 0x6b, 0xd1, 0xa7, 0xeb, 0x89, - 0x72, 0xe2, 0xfd, 0x34, 0x96, 0x05, 0xfd, 0xcf, 0x12, 0x50, 0x30, 0x5c, 0xd7, 0x63, 0xc2, 0x56, - 0x80, 0x10, 0xa4, 0x5d, 0x32, 0xa2, 0x02, 0x94, 0xc7, 0xe2, 0x1b, 0xd5, 0x20, 0xeb, 0x90, 0x03, - 0xea, 0x04, 0xeb, 0xc9, 0x72, 0xea, 0xfd, 0xc2, 0xd6, 0x0f, 0x2a, 0x2f, 0xf6, 0xb9, 0x12, 0x33, - 0x52, 0xd9, 0x15, 0xe8, 0x86, 0xcb, 0xfc, 0x29, 0x56, 0xaa, 0xa5, 0x1f, 0x42, 0x21, 0x26, 0x46, - 0x1a, 0xa4, 0x8e, 0xe8, 0x54, 0x35, 0xc3, 0x3f, 0x79, 0xff, 0x8e, 0x89, 0x33, 0xa1, 0xeb, 0x49, - 0x21, 0x93, 0x85, 0xcf, 0x93, 0xf7, 0x12, 0xfa, 0x97, 0x90, 0xc7, 0x34, 0xf0, 0x26, 0xbe, 0x49, - 0x03, 0xf4, 0x01, 0xe4, 0x5d, 0xe2, 0x7a, 0x7d, 0x73, 0x3c, 0x09, 0x84, 0x7a, 0xaa, 0x5a, 0x3c, - 0x7f, 0xbe, 0x99, 0x6b, 0x11, 0xd7, 0xab, 0x75, 0xf6, 0x03, 0x9c, 0xe3, 0xd5, 0xb5, 0xf1, 0x24, - 0x40, 0x6f, 0x43, 0x71, 0x44, 0x47, 0x9e, 0x3f, 0xed, 0x1f, 0x4c, 0x19, 0x0d, 0x84, 0xe1, 0x14, - 0x2e, 0x48, 0x59, 0x95, 0x8b, 0xf4, 0x3f, 0x4e, 0xc0, 0xb5, 0xd0, 0x36, 0xa6, 0xbf, 0x35, 0xb1, - 0x7d, 0x3a, 0xa2, 0x2e, 0x0b, 0xd0, 0xa7, 0x90, 0x75, 0xec, 0x91, 0xcd, 0x64, 0x1b, 0x85, 0xad, - 0xb7, 0x16, 0x8d, 0x39, 0xea, 0x15, 0x56, 0x60, 0x64, 0x40, 0xd1, 0xa7, 0x01, 0xf5, 0x8f, 0xe5, - 0x4c, 0x88, 0x26, 0x5f, 0xa9, 0x3c, 0xa7, 0xa2, 0x6f, 0x43, 0xae, 0xe3, 0x10, 0x76, 0xe8, 0xf9, - 0x23, 0xa4, 0x43, 0x91, 0xf8, 0xe6, 0xd0, 0x66, 0xd4, 0x64, 0x13, 0x3f, 0x5c, 0x95, 0x39, 0x19, - 0xba, 0x0e, 0x49, 0x4f, 0x36, 0x94, 0xaf, 0x66, 0xcf, 0x9f, 0x6f, 0x26, 0xdb, 0x5d, 0x9c, 0xf4, - 0x02, 0xfd, 0x3e, 0x5c, 0xe9, 0x38, 0x93, 0x81, 0xed, 0xd6, 0x69, 0x60, 0xfa, 0xf6, 0x98, 0x5b, - 0xe7, 0xcb, 0xcb, 0x9d, 0x2f, 0x5c, 0x5e, 0xfe, 0x1d, 0x2d, 0x79, 0x72, 0xb6, 0xe4, 0xfa, 0x1f, - 0x26, 0xe1, 0x4a, 0xc3, 0x1d, 0xd8, 0x2e, 0x8d, 0x6b, 0xdf, 0x82, 0x55, 0x2a, 0x84, 0xfd, 0x63, - 0xe9, 0x54, 0xca, 0xce, 0x8a, 0x94, 0x86, 0x9e, 0xd6, 0xbc, 0xe0, 0x2f, 0x77, 0x17, 0x0d, 0xff, - 0x05, 0xeb, 0x8b, 0xbc, 0x06, 0x35, 0x60, 0x79, 0x2c, 0x06, 0x11, 0xac, 0xa7, 0x84, 0xad, 0x5b, - 0x8b, 0x6c, 0xbd, 0x30, 0xce, 0x6a, 0xfa, 0x9b, 0xe7, 0x9b, 0x4b, 0x38, 0xd4, 0xfd, 0x65, 0x9c, - 0xef, 0xdf, 0x13, 0xb0, 0xd6, 0xf2, 0xac, 0xb9, 0x79, 0x28, 0x41, 0x6e, 0xe8, 0x05, 0x2c, 0xb6, - 0x51, 0xa2, 0x32, 0xba, 0x07, 0xb9, 0xb1, 0x5a, 0x3e, 0xb5, 0xfa, 0x37, 0x17, 0x77, 0x59, 0x62, - 0x70, 0x84, 0x46, 0xf7, 0x21, 0xef, 0x87, 0x3e, 0xb1, 0x9e, 0x7a, 0x1d, 0xc7, 0x99, 0xe1, 0xd1, - 0xaf, 0x41, 0x56, 0x2e, 0xc2, 0x7a, 0x5a, 0x68, 0xde, 0x7a, 0xad, 0x39, 0xc7, 0x4a, 0x49, 0xff, - 0x59, 0x02, 0x34, 0x4c, 0x0e, 0xd9, 0x1e, 0x1d, 0x1d, 0x50, 0xbf, 0xcb, 0x08, 0x9b, 0x04, 0xe8, - 0x3a, 0x64, 0x1d, 0x4a, 0x2c, 0xea, 0x8b, 0x41, 0xe6, 0xb0, 0x2a, 0xa1, 0x7d, 0xee, 0xe4, 0xc4, - 0x1c, 0x92, 0x03, 0xdb, 0xb1, 0xd9, 0x54, 0x0c, 0x73, 0x75, 0xf1, 0x2a, 0x5f, 0xb4, 0x59, 0xc1, - 0x31, 0x45, 0x3c, 0x67, 0x06, 0xad, 0xc3, 0xf2, 0x88, 0x06, 0x01, 0x19, 0x50, 0x31, 0xfa, 0x3c, - 0x0e, 0x8b, 0xfa, 0x7d, 0x28, 0xc6, 0xf5, 0x50, 0x01, 0x96, 0xf7, 0x5b, 0x8f, 0x5a, 0xed, 0x27, - 0x2d, 0x6d, 0x09, 0xad, 0x41, 0x61, 0xbf, 0x85, 0x1b, 0x46, 0x6d, 0xc7, 0xa8, 0xee, 0x36, 0xb4, - 0x04, 0x5a, 0x81, 0xfc, 0xac, 0x98, 0xd4, 0xff, 0x3a, 0x01, 0xc0, 0x17, 0x50, 0x0d, 0xea, 0x73, - 0xc8, 0x04, 0x8c, 0x30, 0xb9, 0x70, 0xab, 0x5b, 0xef, 0x2e, 0xea, 0xf5, 0x0c, 0x5e, 0xe1, 0x7f, - 0x14, 0x4b, 0x95, 0x78, 0x0f, 0x93, 0x73, 0x3d, 0xe4, 0x7b, 0x88, 0x58, 0x96, 0xaf, 0x3a, 0x2e, - 0xbe, 0xf5, 0xfb, 0x90, 0x11, 0xda, 0xf3, 0xdd, 0xcd, 0x41, 0xba, 0xce, 0xbf, 0x12, 0x28, 0x0f, - 0x19, 0xdc, 0x30, 0xea, 0x5f, 0x6a, 0x49, 0xa4, 0x41, 0xb1, 0xde, 0xec, 0xd6, 0xda, 0xad, 0x56, - 0xa3, 0xd6, 0x6b, 0xd4, 0xb5, 0x94, 0x7e, 0x0b, 0x32, 0xcd, 0x11, 0xb7, 0x7c, 0x93, 0x7b, 0xc5, - 0x21, 0xf5, 0xa9, 0x6b, 0x86, 0xce, 0x36, 0x13, 0xe8, 0x3f, 0xcd, 0x43, 0x66, 0xcf, 0x9b, 0xb8, - 0x0c, 0x6d, 0xc5, 0x76, 0xf6, 0xea, 0xd6, 0xc6, 0xa2, 0x61, 0x09, 0x60, 0xa5, 0x37, 0x1d, 0x53, - 0xb5, 0xf3, 0xaf, 0x43, 0x56, 0xfa, 0x8f, 0x1a, 0x8e, 0x2a, 0x71, 0x39, 0x23, 0xfe, 0x80, 0x32, - 0x35, 0x1e, 0x55, 0x42, 0xef, 0x43, 0xce, 0xa7, 0xc4, 0xf2, 0x5c, 0x67, 0x2a, 0xdc, 0x2c, 0x27, - 0x8f, 0x5e, 0x4c, 0x89, 0xd5, 0x76, 0x9d, 0x29, 0x8e, 0x6a, 0xd1, 0x0e, 0x14, 0x0f, 0x6c, 0xd7, - 0xea, 0x7b, 0x63, 0x79, 0x0e, 0x66, 0x5e, 0xee, 0x94, 0xb2, 0x57, 0x55, 0xdb, 0xb5, 0xda, 0x12, - 0x8c, 0x0b, 0x07, 0xb3, 0x02, 0x6a, 0xc1, 0xea, 0xb1, 0xe7, 0x4c, 0x46, 0x34, 0xb2, 0x95, 0x15, - 0xb6, 0xde, 0x7b, 0xb9, 0xad, 0xc7, 0x02, 0x1f, 0x5a, 0x5b, 0x39, 0x8e, 0x17, 0xd1, 0x23, 0x58, - 0x61, 0xa3, 0xf1, 0x61, 0x10, 0x99, 0x5b, 0x16, 0xe6, 0xbe, 0x7f, 0xc9, 0x84, 0x71, 0x78, 0x68, - 0xad, 0xc8, 0x62, 0xa5, 0xd2, 0xef, 0xa7, 0xa0, 0x10, 0xeb, 0x39, 0xea, 0x42, 0x61, 0xec, 0x7b, - 0x63, 0x32, 0x10, 0x67, 0xb9, 0x5a, 0x8b, 0xbb, 0xaf, 0x35, 0xea, 0x4a, 0x67, 0xa6, 0x88, 0xe3, - 0x56, 0xf4, 0xb3, 0x24, 0x14, 0x62, 0x95, 0xe8, 0x36, 0xe4, 0x70, 0x07, 0x37, 0x1f, 0x1b, 0xbd, - 0x86, 0xb6, 0x54, 0xba, 0x79, 0x7a, 0x56, 0x5e, 0x17, 0xd6, 0xe2, 0x06, 0x3a, 0xbe, 0x7d, 0xcc, - 0x5d, 0xef, 0x7d, 0x58, 0x0e, 0xa1, 0x89, 0xd2, 0x9b, 0xa7, 0x67, 0xe5, 0x37, 0x2e, 0x42, 0x63, - 0x48, 0xdc, 0xdd, 0x31, 0x70, 0xa3, 0xae, 0x25, 0x17, 0x23, 0x71, 0x77, 0x48, 0x7c, 0x6a, 0xa1, - 0xef, 0x43, 0x56, 0x01, 0x53, 0xa5, 0xd2, 0xe9, 0x59, 0xf9, 0xfa, 0x45, 0xe0, 0x0c, 0x87, 0xbb, - 0xbb, 0xc6, 0xe3, 0x86, 0x96, 0x5e, 0x8c, 0xc3, 0x5d, 0x87, 0x1c, 0x53, 0xf4, 0x2e, 0x64, 0x24, - 0x2c, 0x53, 0xba, 0x71, 0x7a, 0x56, 0xfe, 0xde, 0x0b, 0xe6, 0x38, 0xaa, 0xb4, 0xfe, 0x47, 0x7f, - 0xbe, 0xb1, 0xf4, 0xb7, 0x7f, 0xb1, 0xa1, 0x5d, 0xac, 0x2e, 0xfd, 0x77, 0x02, 0x56, 0xe6, 0x96, - 0x1c, 0xe9, 0x90, 0x75, 0x3d, 0xd3, 0x1b, 0xcb, 0x23, 0x3e, 0x57, 0x85, 0xf3, 0xe7, 0x9b, 0xd9, - 0x96, 0x57, 0xf3, 0xc6, 0x53, 0xac, 0x6a, 0xd0, 0xa3, 0x0b, 0x97, 0xd4, 0xc7, 0xaf, 0xe9, 0x4f, - 0x0b, 0xaf, 0xa9, 0x07, 0xb0, 0x62, 0xf9, 0xf6, 0x31, 0xf5, 0xfb, 0xa6, 0xe7, 0x1e, 0xda, 0x03, - 0x75, 0x7c, 0x97, 0x16, 0xd9, 0xac, 0x0b, 0x20, 0x2e, 0x4a, 0x85, 0x9a, 0xc0, 0xff, 0x12, 0x17, - 0x54, 0xe9, 0x31, 0x14, 0xe3, 0x1e, 0x8a, 0xde, 0x02, 0x08, 0xec, 0xdf, 0xa6, 0x8a, 0xf3, 0x08, - 0x86, 0x84, 0xf3, 0x5c, 0x22, 0x18, 0x0f, 0x7a, 0x0f, 0xd2, 0x23, 0xcf, 0x92, 0x76, 0x56, 0xaa, - 0x57, 0xf9, 0x3d, 0xf9, 0xcf, 0xcf, 0x37, 0x0b, 0x5e, 0x50, 0xd9, 0xb6, 0x1d, 0xba, 0xe7, 0x59, - 0x14, 0x0b, 0x80, 0x7e, 0x0c, 0x69, 0x7e, 0x54, 0xa0, 0x37, 0x21, 0x5d, 0x6d, 0xb6, 0xea, 0xda, - 0x52, 0xe9, 0xca, 0xe9, 0x59, 0x79, 0x45, 0x4c, 0x09, 0xaf, 0xe0, 0xbe, 0x8b, 0x36, 0x21, 0xfb, - 0xb8, 0xbd, 0xbb, 0xbf, 0xc7, 0xdd, 0xeb, 0xea, 0xe9, 0x59, 0x79, 0x2d, 0xaa, 0x96, 0x93, 0x86, - 0xde, 0x82, 0x4c, 0x6f, 0xaf, 0xb3, 0xdd, 0xd5, 0x92, 0x25, 0x74, 0x7a, 0x56, 0x5e, 0x8d, 0xea, - 0x45, 0x9f, 0x4b, 0x57, 0xd4, 0xaa, 0xe6, 0x23, 0xb9, 0xfe, 0x5f, 0x49, 0x58, 0xc1, 0x9c, 0xf3, - 0xfa, 0xac, 0xe3, 0x39, 0xb6, 0x39, 0x45, 0x1d, 0xc8, 0x9b, 0x9e, 0x6b, 0xd9, 0xb1, 0x3d, 0xb5, - 0xf5, 0x92, 0x8b, 0x71, 0xa6, 0x15, 0x96, 0x6a, 0xa1, 0x26, 0x9e, 0x19, 0x41, 0x5b, 0x90, 0xb1, - 0xa8, 0x43, 0xa6, 0x97, 0xdd, 0xd0, 0x75, 0xc5, 0xaf, 0xb1, 0x84, 0x0a, 0x36, 0x49, 0x9e, 0xf6, - 0x09, 0x63, 0x74, 0x34, 0x66, 0xf2, 0x86, 0x4e, 0xe3, 0xc2, 0x88, 0x3c, 0x35, 0x94, 0x08, 0x7d, - 0x02, 0xd9, 0x13, 0xdb, 0xb5, 0xbc, 0x13, 0x75, 0x09, 0x5f, 0x6e, 0x57, 0x61, 0xf5, 0x53, 0x7e, - 0xf7, 0x5e, 0xe8, 0x2c, 0x9f, 0xf5, 0x56, 0xbb, 0xd5, 0x08, 0x67, 0x5d, 0xd5, 0xb7, 0xdd, 0x96, - 0xe7, 0xf2, 0x1d, 0x03, 0xed, 0x56, 0x7f, 0xdb, 0x68, 0xee, 0xee, 0x63, 0x3e, 0xf3, 0xd7, 0x4e, - 0xcf, 0xca, 0x5a, 0x04, 0xd9, 0x26, 0xb6, 0xc3, 0x89, 0xe1, 0x0d, 0x48, 0x19, 0xad, 0x2f, 0xb5, - 0x64, 0x49, 0x3b, 0x3d, 0x2b, 0x17, 0xa3, 0x6a, 0xc3, 0x9d, 0xce, 0x36, 0xd3, 0xc5, 0x76, 0xf5, - 0x7f, 0x4d, 0x42, 0x71, 0x7f, 0x6c, 0x11, 0x46, 0xa5, 0x67, 0xa2, 0x32, 0x14, 0xc6, 0xc4, 0x27, - 0x8e, 0x43, 0x1d, 0x3b, 0x18, 0xa9, 0xe0, 0x21, 0x2e, 0x42, 0xf7, 0xbe, 0xc3, 0x64, 0x2a, 0x62, - 0xa6, 0xa6, 0x74, 0x1f, 0x56, 0x0f, 0x65, 0x67, 0xfb, 0xc4, 0x14, 0xab, 0x9b, 0x12, 0xab, 0x5b, - 0x59, 0x64, 0x22, 0xde, 0xab, 0x8a, 0x1a, 0xa3, 0x21, 0xb4, 0xf0, 0xca, 0x61, 0xbc, 0x88, 0x3e, - 0x83, 0xe5, 0x91, 0xe7, 0xda, 0xcc, 0xf3, 0x5f, 0x6b, 0x1d, 0x42, 0x30, 0xba, 0x0d, 0x57, 0xf8, - 0x0a, 0x87, 0x5d, 0x12, 0xd5, 0xe2, 0xe6, 0x4a, 0xe2, 0xb5, 0x11, 0x79, 0xaa, 0xda, 0xc4, 0x5c, - 0xac, 0x7f, 0x06, 0x2b, 0x73, 0x7d, 0xe0, 0xb7, 0x79, 0xc7, 0xd8, 0xef, 0x36, 0xb4, 0x25, 0x54, - 0x84, 0x5c, 0xad, 0xdd, 0xea, 0x35, 0x5b, 0xfb, 0x9c, 0x8e, 0x14, 0x21, 0x87, 0xdb, 0xbb, 0xbb, - 0x55, 0xa3, 0xf6, 0x48, 0x4b, 0xea, 0xbf, 0x88, 0xe6, 0x57, 0xf1, 0x91, 0xea, 0x3c, 0x1f, 0xf9, - 0xf0, 0xe5, 0x43, 0x57, 0x8c, 0x64, 0x56, 0x88, 0x78, 0xc9, 0xff, 0x07, 0x10, 0xcb, 0x48, 0xad, - 0x3e, 0x61, 0x97, 0xc5, 0x1c, 0xbd, 0x30, 0x9a, 0xc4, 0x79, 0xa5, 0x60, 0x30, 0xf4, 0x05, 0x14, - 0x4d, 0x6f, 0x34, 0x76, 0xa8, 0xd2, 0x4f, 0xbd, 0x8e, 0x7e, 0x21, 0x52, 0x31, 0x58, 0x9c, 0x17, - 0xa5, 0xe7, 0x99, 0xdb, 0x1f, 0x24, 0xa0, 0x10, 0xeb, 0xf0, 0x3c, 0x15, 0x2a, 0x42, 0x6e, 0xbf, - 0x53, 0x37, 0x7a, 0xcd, 0xd6, 0x43, 0x2d, 0x81, 0x00, 0xb2, 0x62, 0x02, 0xeb, 0x5a, 0x92, 0x53, - 0xb8, 0x5a, 0x7b, 0xaf, 0xb3, 0xdb, 0x10, 0x64, 0x08, 0x5d, 0x03, 0x2d, 0x9c, 0xc2, 0x7e, 0xb7, - 0x67, 0x60, 0x2e, 0x4d, 0xa3, 0xab, 0xb0, 0x16, 0x49, 0x95, 0x66, 0x06, 0x5d, 0x07, 0x14, 0x09, - 0x67, 0x26, 0xb2, 0xfa, 0xef, 0xc2, 0x5a, 0xcd, 0x73, 0x19, 0xb1, 0xdd, 0x88, 0xde, 0x6e, 0xf1, - 0x71, 0x2b, 0x51, 0xdf, 0xb6, 0xe4, 0x69, 0x5b, 0x5d, 0x3b, 0x7f, 0xbe, 0x59, 0x88, 0xa0, 0xcd, - 0x3a, 0x1f, 0x69, 0x58, 0xb0, 0xf8, 0x9e, 0x1a, 0xdb, 0x96, 0x98, 0xe2, 0x4c, 0x75, 0xf9, 0xfc, - 0xf9, 0x66, 0xaa, 0xd3, 0xac, 0x63, 0x2e, 0x43, 0x6f, 0x42, 0x9e, 0x3e, 0xb5, 0x59, 0xdf, 0xe4, - 0xa7, 0x2b, 0x9f, 0xc3, 0x0c, 0xce, 0x71, 0x41, 0x8d, 0x1f, 0xa6, 0x55, 0x80, 0x8e, 0xe7, 0x33, - 0xd5, 0xf2, 0x27, 0x90, 0x19, 0x7b, 0xbe, 0x88, 0x2d, 0xf9, 0xd5, 0xb3, 0x90, 0xac, 0x71, 0xb8, - 0x74, 0x76, 0x2c, 0xc1, 0xfa, 0xdf, 0x25, 0x01, 0x7a, 0x24, 0x38, 0x52, 0x46, 0xee, 0x43, 0x3e, - 0x4a, 0x0e, 0x5c, 0x16, 0xa4, 0xc6, 0xd6, 0x3c, 0xc2, 0xa3, 0x8f, 0x43, 0xaf, 0x93, 0xdc, 0x7d, - 0xb1, 0xa2, 0x6a, 0x6b, 0x11, 0xfd, 0x9d, 0x27, 0xe8, 0xfc, 0xbe, 0xa2, 0xbe, 0xaf, 0x16, 0x9f, - 0x7f, 0xa2, 0x9a, 0x38, 0xb3, 0xe5, 0xbc, 0x29, 0xf6, 0xf7, 0xce, 0xa2, 0x46, 0x2e, 0x2c, 0xca, - 0xce, 0x12, 0x9e, 0xe9, 0xa1, 0x07, 0x50, 0xe0, 0x43, 0xef, 0x07, 0xa2, 0x4e, 0x11, 0xbf, 0x97, - 0xce, 0x96, 0xb4, 0x80, 0x61, 0x1c, 0x7d, 0x57, 0x35, 0x58, 0xf5, 0x27, 0x2e, 0x1f, 0xb6, 0xb2, - 0xa1, 0xdb, 0xf0, 0x46, 0x8b, 0xb2, 0x13, 0xcf, 0x3f, 0x32, 0x18, 0x23, 0xe6, 0x90, 0x47, 0xfb, - 0xea, 0xa4, 0x9b, 0xb1, 0xde, 0xc4, 0x1c, 0xeb, 0x5d, 0x87, 0x65, 0xe2, 0xd8, 0x24, 0xa0, 0x92, - 0x2a, 0xe4, 0x71, 0x58, 0xe4, 0xdc, 0x9c, 0x33, 0x7d, 0x1a, 0x04, 0x54, 0xc6, 0xa7, 0x79, 0x3c, - 0x13, 0xe8, 0xff, 0x98, 0x04, 0x68, 0x76, 0x8c, 0x3d, 0x65, 0xbe, 0x0e, 0xd9, 0x43, 0x32, 0xb2, - 0x9d, 0xe9, 0x65, 0x3b, 0x7d, 0x86, 0xaf, 0x18, 0xd2, 0xd0, 0xb6, 0xd0, 0xc1, 0x4a, 0x57, 0x50, - 0xf6, 0xc9, 0x81, 0x4b, 0x59, 0x44, 0xd9, 0x45, 0x89, 0xf3, 0x03, 0x9f, 0xb8, 0xd1, 0xca, 0xc8, - 0x02, 0xef, 0xfa, 0x80, 0x30, 0x7a, 0x42, 0xa6, 0xe1, 0xc6, 0x54, 0x45, 0xb4, 0xc3, 0xa9, 0x7c, - 0x40, 0xfd, 0x63, 0x6a, 0xad, 0x67, 0x84, 0x17, 0xbe, 0xaa, 0x3f, 0x58, 0xc1, 0x25, 0xf3, 0x89, - 0xb4, 0x4b, 0xf7, 0xc5, 0x75, 0x3d, 0xab, 0xfa, 0x4e, 0xd1, 0xf5, 0x1d, 0x58, 0x99, 0x1b, 0xe7, - 0x0b, 0xb1, 0x52, 0xb3, 0xf3, 0xf8, 0x13, 0x2d, 0xad, 0xbe, 0x3e, 0xd3, 0xb2, 0xfa, 0x5f, 0xa5, - 0xe4, 0x56, 0x52, 0xb3, 0xba, 0x38, 0x5f, 0x95, 0x13, 0xd9, 0x2f, 0xd3, 0x73, 0x94, 0x7f, 0xbf, - 0x77, 0xf9, 0x0e, 0xe3, 0xdc, 0x5b, 0xc0, 0x71, 0xa4, 0x88, 0x36, 0xa1, 0x20, 0xd7, 0xbf, 0xcf, - 0xfd, 0x49, 0x4c, 0xeb, 0x0a, 0x06, 0x29, 0xe2, 0x9a, 0xe8, 0x16, 0xac, 0x8e, 0x27, 0x07, 0x8e, - 0x1d, 0x0c, 0xa9, 0x25, 0x31, 0x69, 0x81, 0x59, 0x89, 0xa4, 0x02, 0xb6, 0x07, 0x45, 0x25, 0xe8, - 0x0b, 0xde, 0x95, 0x11, 0x1d, 0xba, 0xfd, 0xaa, 0x0e, 0x49, 0x15, 0x41, 0xc7, 0x0a, 0xe3, 0x59, - 0x41, 0xaf, 0x43, 0x2e, 0xec, 0x2c, 0x5a, 0x87, 0x54, 0xaf, 0xd6, 0xd1, 0x96, 0x4a, 0x6b, 0xa7, - 0x67, 0xe5, 0x42, 0x28, 0xee, 0xd5, 0x3a, 0xbc, 0x66, 0xbf, 0xde, 0xd1, 0x12, 0xf3, 0x35, 0xfb, - 0xf5, 0x4e, 0x29, 0xcd, 0x6f, 0x7e, 0xfd, 0x10, 0x0a, 0xb1, 0x16, 0xd0, 0x3b, 0xb0, 0xdc, 0x6c, - 0x3d, 0xc4, 0x8d, 0x6e, 0x57, 0x5b, 0x2a, 0x5d, 0x3f, 0x3d, 0x2b, 0xa3, 0x58, 0x6d, 0xd3, 0x1d, - 0xf0, 0xf5, 0x41, 0x6f, 0x41, 0x7a, 0xa7, 0xdd, 0xed, 0x85, 0x44, 0x2f, 0x86, 0xd8, 0xf1, 0x02, - 0x56, 0xba, 0xaa, 0x28, 0x45, 0xdc, 0xb0, 0xfe, 0x27, 0x09, 0xc8, 0x4a, 0xbe, 0xbb, 0x70, 0xa1, - 0x0c, 0x58, 0x0e, 0xa3, 0x30, 0x49, 0xc2, 0xdf, 0x7b, 0x39, 0x61, 0xae, 0x28, 0x7e, 0x2b, 0xdd, - 0x2f, 0xd4, 0x2b, 0x7d, 0x0e, 0xc5, 0x78, 0xc5, 0x77, 0x72, 0xbe, 0xdf, 0x81, 0x02, 0xf7, 0xef, - 0x90, 0x38, 0x6f, 0x41, 0x56, 0x72, 0x72, 0x75, 0x9a, 0x5e, 0xc6, 0xde, 0x15, 0x12, 0xdd, 0x83, - 0x65, 0xc9, 0xf8, 0xc3, 0xfc, 0xd4, 0xc6, 0xe5, 0xbb, 0x08, 0x87, 0x70, 0xfd, 0x01, 0xa4, 0x3b, - 0x94, 0xfa, 0x7c, 0xee, 0x5d, 0xcf, 0xa2, 0xb3, 0x0b, 0x48, 0x05, 0x2b, 0x16, 0x6d, 0xd6, 0x79, - 0xb0, 0x62, 0xd1, 0xa6, 0x15, 0xa5, 0x17, 0x92, 0xb1, 0xf4, 0x42, 0x0f, 0x8a, 0x4f, 0xa8, 0x3d, - 0x18, 0x32, 0x6a, 0x09, 0x43, 0x1f, 0x42, 0x7a, 0x4c, 0xa3, 0xce, 0xaf, 0x2f, 0x74, 0x30, 0x4a, - 0x7d, 0x2c, 0x50, 0xfc, 0x1c, 0x39, 0x11, 0xda, 0x2a, 0x2b, 0xaa, 0x4a, 0xfa, 0x3f, 0x24, 0x61, - 0xb5, 0x19, 0x04, 0x13, 0xe2, 0x9a, 0x21, 0x43, 0xf9, 0xd1, 0x3c, 0x43, 0x79, 0x7f, 0xe1, 0x08, - 0xe7, 0x54, 0xe6, 0xb3, 0x26, 0xea, 0x72, 0x48, 0x46, 0x97, 0x83, 0xfe, 0x1f, 0x89, 0x30, 0x35, - 0x72, 0x2b, 0xb6, 0xdd, 0x4b, 0xeb, 0xa7, 0x67, 0xe5, 0x6b, 0x71, 0x4b, 0x74, 0xdf, 0x3d, 0x72, - 0xbd, 0x13, 0x17, 0xbd, 0x0d, 0x19, 0xdc, 0x68, 0x35, 0x9e, 0x68, 0x09, 0xe9, 0x9e, 0x73, 0x20, - 0x4c, 0x5d, 0x7a, 0xc2, 0x2d, 0x75, 0x1a, 0xad, 0x3a, 0xe7, 0x12, 0xc9, 0x05, 0x96, 0x3a, 0xd4, - 0xb5, 0x6c, 0x77, 0x80, 0xde, 0x81, 0x6c, 0xb3, 0xdb, 0xdd, 0x17, 0xc1, 0xeb, 0x1b, 0xa7, 0x67, - 0xe5, 0xab, 0x73, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x02, 0x10, 0xe7, - 0x7d, 0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0x16, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb, - 0x97, 0x24, 0x68, 0x86, 0x69, 0xd2, 0x31, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xcc, 0xbf, - 0x6c, 0x1a, 0xf2, 0x80, 0x7b, 0x0b, 0xf3, 0xea, 0x17, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x8d, - 0xec, 0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0xba, 0x00, 0x81, - 0xee, 0x40, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0x6f, 0xbe, 0x2c, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2, - 0x0d, 0x00, 0x32, 0x61, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x81, 0x6c, - 0x40, 0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xc1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9, - 0x52, 0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde, - 0x44, 0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x65, 0xd4, 0x77, - 0x89, 0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0xb0, 0x30, 0x17, 0x1a, 0x69, 0x54, - 0x6a, 0xc6, 0x82, 0xf3, 0xff, 0x06, 0xa4, 0x26, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe3, - 0x5d, 0xcc, 0x65, 0xa8, 0x31, 0x3b, 0xb6, 0x52, 0x2f, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f, - 0xba, 0x3e, 0x04, 0x98, 0xf5, 0x1a, 0x6d, 0x40, 0xa6, 0xb6, 0xdd, 0xed, 0xee, 0x6a, 0x4b, 0xf2, - 0x6c, 0x9e, 0x55, 0x09, 0xb1, 0xfe, 0x97, 0x09, 0xc8, 0xd5, 0x0c, 0x75, 0x63, 0x6e, 0x83, 0x26, - 0x0e, 0x1c, 0x93, 0xfa, 0xac, 0x4f, 0x9f, 0x8e, 0x6d, 0x7f, 0xaa, 0xce, 0x8c, 0xcb, 0xc3, 0xa4, - 0x55, 0xae, 0x55, 0xa3, 0x3e, 0x6b, 0x08, 0x1d, 0x84, 0xa1, 0x48, 0xd5, 0x10, 0xfb, 0x26, 0x09, - 0x4f, 0xf0, 0x8d, 0xcb, 0xa7, 0x42, 0xd2, 0xeb, 0x59, 0x39, 0xc0, 0x85, 0xd0, 0x48, 0x8d, 0x04, - 0xfa, 0x63, 0xb8, 0xda, 0xf6, 0xcd, 0x21, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x01, 0xdc, 0x64, - 0x24, 0x38, 0xea, 0x0f, 0xed, 0x80, 0x79, 0xfe, 0xb4, 0xef, 0x53, 0x46, 0x5d, 0x5e, 0xdf, 0x17, - 0xcf, 0x2e, 0x2a, 0xc9, 0x71, 0x83, 0x63, 0x76, 0x24, 0x04, 0x87, 0x88, 0x5d, 0x0e, 0xd0, 0x9b, - 0x50, 0xe4, 0x6c, 0xb6, 0x4e, 0x0f, 0xc9, 0xc4, 0x61, 0x01, 0xfa, 0x21, 0x80, 0xe3, 0x0d, 0xfa, - 0xaf, 0x7d, 0xdc, 0xe7, 0x1d, 0x6f, 0x20, 0x3f, 0xf5, 0xdf, 0x00, 0xad, 0x6e, 0x07, 0x63, 0xc2, - 0xcc, 0x61, 0x98, 0xbd, 0x41, 0x0f, 0x41, 0x1b, 0x52, 0xe2, 0xb3, 0x03, 0x4a, 0x58, 0x7f, 0x4c, - 0x7d, 0xdb, 0xb3, 0x5e, 0x6b, 0x4a, 0xd7, 0x22, 0xad, 0x8e, 0x50, 0xd2, 0xff, 0x33, 0x01, 0x80, - 0xc9, 0x61, 0x48, 0x6e, 0x7e, 0x00, 0x57, 0x02, 0x97, 0x8c, 0x83, 0xa1, 0xc7, 0xfa, 0xb6, 0xcb, - 0xa8, 0x7f, 0x4c, 0x1c, 0x15, 0x81, 0x6b, 0x61, 0x45, 0x53, 0xc9, 0xd1, 0x87, 0x80, 0x8e, 0x28, - 0x1d, 0xf7, 0x3d, 0xc7, 0xea, 0x87, 0x95, 0xf2, 0x5d, 0x28, 0x8d, 0x35, 0x5e, 0xd3, 0x76, 0xac, - 0x6e, 0x28, 0x47, 0x55, 0xd8, 0xe0, 0x33, 0x40, 0x5d, 0xe6, 0xdb, 0x34, 0xe8, 0x1f, 0x7a, 0x7e, - 0x3f, 0x70, 0xbc, 0x93, 0xfe, 0xa1, 0xe7, 0x38, 0xde, 0x09, 0xf5, 0xc3, 0xfc, 0x46, 0xc9, 0xf1, - 0x06, 0x0d, 0x09, 0xda, 0xf6, 0xfc, 0xae, 0xe3, 0x9d, 0x6c, 0x87, 0x08, 0xce, 0x80, 0x66, 0xc3, - 0x66, 0xb6, 0x79, 0x14, 0x32, 0xa0, 0x48, 0xda, 0xb3, 0xcd, 0x23, 0xf4, 0x0e, 0xac, 0x50, 0x87, - 0x8a, 0x28, 0x59, 0xa2, 0x32, 0x02, 0x55, 0x0c, 0x85, 0x1c, 0xa4, 0xff, 0x3f, 0xc8, 0x77, 0x1c, - 0x62, 0x8a, 0xd7, 0x37, 0x54, 0x06, 0x1e, 0x74, 0x71, 0x27, 0xb0, 0x5d, 0x15, 0x25, 0xe5, 0x71, - 0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x23, 0xea, 0x8a, 0x87, 0x0a, 0xce, - 0xe8, 0xd5, 0x52, 0xe6, 0xb1, 0x2a, 0x89, 0x80, 0x85, 0xb8, 0x64, 0x40, 0xfd, 0x28, 0x5f, 0x2f, - 0x8b, 0xfa, 0x37, 0x09, 0xc8, 0x62, 0xcf, 0x63, 0x35, 0x03, 0x95, 0x21, 0x6b, 0x92, 0x7e, 0xb8, - 0xf3, 0x8a, 0xd5, 0xfc, 0xf9, 0xf3, 0xcd, 0x4c, 0xcd, 0x78, 0x44, 0xa7, 0x38, 0x63, 0x92, 0x47, - 0x74, 0xca, 0xaf, 0x68, 0x93, 0x88, 0xfd, 0x22, 0xcc, 0x14, 0xe5, 0x15, 0x5d, 0x33, 0xf8, 0x66, - 0xc0, 0x59, 0x93, 0xf0, 0x7f, 0x74, 0x07, 0x8a, 0x0a, 0xd4, 0x1f, 0x92, 0x60, 0x28, 0x79, 0x78, - 0x75, 0xf5, 0xfc, 0xf9, 0x26, 0x48, 0xe4, 0x0e, 0x09, 0x86, 0x18, 0x24, 0x9a, 0x7f, 0xa3, 0x06, - 0x14, 0xbe, 0xf2, 0x6c, 0xb7, 0xcf, 0xc4, 0x20, 0x54, 0xaa, 0x62, 0xe1, 0xfe, 0x99, 0x0d, 0x55, - 0xe5, 0x4f, 0xe0, 0xab, 0x48, 0xa2, 0xff, 0x53, 0x02, 0x0a, 0xdc, 0xa6, 0x7d, 0x68, 0x9b, 0xfc, - 0x4a, 0xfd, 0xee, 0x27, 0xfd, 0x0d, 0x48, 0x99, 0x81, 0xaf, 0xc6, 0x26, 0x8e, 0xba, 0x5a, 0x17, - 0x63, 0x2e, 0x43, 0x5f, 0x40, 0x56, 0x05, 0x5f, 0xf2, 0x90, 0xd7, 0x5f, 0x7d, 0xf9, 0xab, 0x2e, - 0x2a, 0x3d, 0xb1, 0x96, 0xb3, 0xde, 0x89, 0x51, 0x16, 0x71, 0x5c, 0x84, 0xae, 0x43, 0xd2, 0x74, - 0x85, 0x53, 0xa8, 0x07, 0xcc, 0x5a, 0x0b, 0x27, 0x4d, 0x57, 0xff, 0xfb, 0x04, 0xac, 0x34, 0x5c, - 0xd3, 0x9f, 0x8a, 0x43, 0x92, 0x2f, 0xc4, 0x4d, 0xc8, 0x07, 0x93, 0x83, 0x60, 0x1a, 0x30, 0x3a, - 0x0a, 0xdf, 0x42, 0x22, 0x01, 0x6a, 0x42, 0x9e, 0x38, 0x03, 0xcf, 0xb7, 0xd9, 0x70, 0xa4, 0x78, - 0xff, 0xe2, 0x83, 0x39, 0x6e, 0xb3, 0x62, 0x84, 0x2a, 0x78, 0xa6, 0x1d, 0x1e, 0xc5, 0x29, 0xd1, - 0x59, 0x71, 0x14, 0xbf, 0x0d, 0x45, 0x87, 0x8c, 0x44, 0x34, 0xca, 0xc3, 0x49, 0x31, 0x8e, 0x34, - 0x2e, 0x28, 0x19, 0x8f, 0xb1, 0x75, 0x1d, 0xf2, 0x91, 0x31, 0xb4, 0x06, 0x05, 0xa3, 0xd1, 0xed, - 0xdf, 0xdd, 0xba, 0xd7, 0x7f, 0x58, 0xdb, 0xd3, 0x96, 0x14, 0x13, 0xf8, 0x9b, 0x04, 0xac, 0xec, - 0x49, 0x1f, 0x54, 0xec, 0xea, 0x1d, 0x58, 0xf6, 0xc9, 0x21, 0x0b, 0xf9, 0x5f, 0x5a, 0x3a, 0x17, - 0x3f, 0x04, 0x38, 0xff, 0xe3, 0x55, 0x8b, 0xf9, 0x5f, 0xec, 0x75, 0x2e, 0x75, 0xe9, 0xeb, 0x5c, - 0xfa, 0x57, 0xf2, 0x3a, 0xa7, 0xff, 0x24, 0x09, 0x6b, 0xea, 0xa2, 0x0e, 0x5f, 0x9f, 0xd0, 0x07, - 0x90, 0x97, 0x77, 0xf6, 0x8c, 0xbd, 0x8a, 0x07, 0x21, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37, - 0x2d, 0x1e, 0x4e, 0x29, 0x68, 0xec, 0xad, 0x19, 0xa4, 0xa8, 0xc5, 0x63, 0x81, 0x3a, 0xa4, 0x0f, - 0x6d, 0x87, 0x2a, 0x3f, 0x5b, 0x98, 0x01, 0xbc, 0xd0, 0xbc, 0x48, 0x58, 0xf7, 0x44, 0x40, 0xb6, - 0xb3, 0x84, 0x85, 0x76, 0xe9, 0xf7, 0x00, 0x66, 0xd2, 0x85, 0x31, 0x07, 0xbf, 0xd7, 0x55, 0x06, - 0x27, 0xbc, 0xd7, 0x9b, 0x75, 0xcc, 0x65, 0xbc, 0x6a, 0x60, 0x5b, 0x6a, 0xe7, 0x8a, 0xaa, 0x87, - 0xbc, 0x6a, 0x60, 0x5b, 0x51, 0xd6, 0x3c, 0xfd, 0x8a, 0xac, 0x79, 0x35, 0x17, 0x26, 0x11, 0xf4, - 0x36, 0x5c, 0xaf, 0x3a, 0xc4, 0x3c, 0x72, 0xec, 0x80, 0x51, 0x2b, 0xbe, 0x43, 0x3f, 0x85, 0xec, - 0xdc, 0xbd, 0xfb, 0x8a, 0xb4, 0x8d, 0x02, 0xeb, 0x3f, 0x49, 0x40, 0x71, 0x87, 0x12, 0x87, 0x0d, - 0x67, 0xb1, 0x2f, 0xa3, 0x01, 0x53, 0xe7, 0xa3, 0xf8, 0x46, 0xf7, 0x20, 0x17, 0xdd, 0x14, 0xaf, - 0x93, 0xdc, 0x8e, 0xd0, 0xe8, 0x33, 0x58, 0xe6, 0x9e, 0xed, 0x4d, 0x42, 0x42, 0xf7, 0x8a, 0xac, - 0xa9, 0x02, 0xf3, 0x43, 0xd6, 0xa7, 0xe2, 0x82, 0x10, 0xb3, 0x93, 0xc1, 0x61, 0x51, 0xff, 0x9f, - 0x04, 0x5c, 0xdb, 0x23, 0xd3, 0x03, 0xaa, 0x76, 0x1c, 0xb5, 0x30, 0x35, 0x3d, 0xdf, 0x42, 0x9d, - 0xf8, 0x4e, 0xbd, 0x24, 0xa1, 0xbf, 0x48, 0x79, 0xf1, 0x86, 0x0d, 0x99, 0x62, 0x32, 0xc6, 0x14, - 0xaf, 0x41, 0xc6, 0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28, - 0xcb, 0x2e, 0x72, 0xe4, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x2f, 0xa0, 0xd4, 0x6d, 0xd4, 0x70, 0xa3, - 0x57, 0x6d, 0xff, 0x7a, 0xbf, 0x6b, 0xec, 0x76, 0x8d, 0xad, 0x3b, 0xfd, 0x4e, 0x7b, 0xf7, 0xcb, - 0xbb, 0x1f, 0xdf, 0xf9, 0x54, 0x4b, 0x94, 0xca, 0xa7, 0x67, 0xe5, 0x9b, 0x2d, 0xa3, 0xb6, 0x2b, - 0xdd, 0xf2, 0xc0, 0x7b, 0xda, 0x25, 0x4e, 0x40, 0xb6, 0xee, 0x74, 0x3c, 0x67, 0xca, 0x31, 0xb7, - 0x7f, 0x91, 0x82, 0x7c, 0x94, 0x44, 0xe3, 0xde, 0xc5, 0x23, 0x18, 0xd5, 0x54, 0x24, 0x6f, 0xd1, - 0x13, 0xf4, 0xf6, 0x2c, 0x76, 0xf9, 0x42, 0x26, 0xf3, 0xa3, 0xea, 0x30, 0x6e, 0x79, 0x17, 0x72, - 0x46, 0xb7, 0xdb, 0x7c, 0xd8, 0x6a, 0xd4, 0xb5, 0xaf, 0x13, 0xa5, 0xef, 0x9d, 0x9e, 0x95, 0xaf, - 0x44, 0x20, 0x23, 0x08, 0xec, 0x81, 0x4b, 0x2d, 0x81, 0xaa, 0xd5, 0x1a, 0x9d, 0x5e, 0xa3, 0xae, - 0x3d, 0x4b, 0x5e, 0x44, 0x09, 0x2e, 0x2e, 0x1e, 0xe6, 0xf2, 0x1d, 0xdc, 0xe8, 0x18, 0x98, 0x37, - 0xf8, 0x75, 0x52, 0x86, 0x54, 0xb3, 0x16, 0x7d, 0x3a, 0x26, 0x3e, 0x6f, 0x73, 0x23, 0x7c, 0xa0, - 0x7e, 0x96, 0x92, 0x8f, 0x37, 0xb3, 0x8c, 0x20, 0x25, 0xd6, 0x94, 0xb7, 0x26, 0xb2, 0xb1, 0xc2, - 0x4c, 0xea, 0x42, 0x6b, 0x5d, 0x46, 0x7c, 0xc6, 0xad, 0xe8, 0xb0, 0x8c, 0xf7, 0x5b, 0x2d, 0x0e, - 0x7a, 0x96, 0xbe, 0x30, 0x3a, 0x3c, 0x71, 0x5d, 0x8e, 0xb9, 0x05, 0xb9, 0x30, 0x59, 0xab, 0x7d, - 0x9d, 0xbe, 0xd0, 0xa1, 0x5a, 0x98, 0x69, 0x16, 0x0d, 0xee, 0xec, 0xf7, 0xc4, 0xfb, 0xf9, 0xb3, - 0xcc, 0xc5, 0x06, 0x87, 0x13, 0x66, 0xf1, 0x60, 0xb1, 0x1c, 0x45, 0x6f, 0x5f, 0x67, 0x24, 0x1f, - 0x8e, 0x30, 0x2a, 0x74, 0x7b, 0x17, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0xd4, 0xfe, 0x2c, 0x7b, 0xc1, - 0x0e, 0xa6, 0x5f, 0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x1d, 0x43, 0x4c, 0xf9, 0x45, 0x54, 0xdb, - 0x1f, 0x0f, 0x89, 0x4b, 0xad, 0xd9, 0x0b, 0x56, 0x54, 0x75, 0xfb, 0x37, 0x21, 0x17, 0xde, 0xb0, - 0x68, 0x03, 0xb2, 0x4f, 0xda, 0xf8, 0x51, 0x03, 0x6b, 0x4b, 0x72, 0x0e, 0xc3, 0x9a, 0x27, 0x92, - 0xa2, 0x94, 0x61, 0x79, 0xcf, 0x68, 0x19, 0x0f, 0x1b, 0x38, 0x4c, 0xac, 0x84, 0x00, 0x75, 0x4d, - 0x94, 0x34, 0xd5, 0x40, 0x64, 0xb3, 0x7a, 0xf3, 0x9b, 0x6f, 0x37, 0x96, 0x7e, 0xf6, 0xed, 0xc6, - 0xd2, 0xcf, 0xbf, 0xdd, 0x48, 0x3c, 0x3b, 0xdf, 0x48, 0x7c, 0x73, 0xbe, 0x91, 0xf8, 0xe9, 0xf9, - 0x46, 0xe2, 0xdf, 0xce, 0x37, 0x12, 0x07, 0x59, 0x11, 0xc2, 0x7c, 0xfc, 0xbf, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x6b, 0x1c, 0x13, 0xe7, 0x66, 0x26, 0x00, 0x00, + 0x76, 0xbf, 0xf8, 0x29, 0xf2, 0x91, 0x92, 0xda, 0x65, 0xaf, 0x47, 0xe6, 0x78, 0x24, 0x4e, 0x7b, + 0xbc, 0xe3, 0xf1, 0xfa, 0xcf, 0xb1, 0x35, 0x1f, 0xf0, 0x8e, 0xff, 0x59, 0xbb, 0xf9, 0x21, 0x8b, + 0x6b, 0x89, 0x24, 0x8a, 0x94, 0x9d, 0x41, 0x80, 0x10, 0xa5, 0xee, 0x12, 0xd5, 0xa3, 0x66, 0x37, + 0xd3, 0x5d, 0x94, 0xcc, 0x04, 0x41, 0x8c, 0x1c, 0x92, 0x40, 0xa7, 0xdc, 0x03, 0x21, 0x08, 0x12, + 0xe4, 0x90, 0xc3, 0x5e, 0x72, 0x08, 0x90, 0xd3, 0x20, 0xa7, 0x39, 0x6e, 0x12, 0x20, 0x58, 0x24, + 0x88, 0x91, 0x51, 0xce, 0x01, 0xf6, 0xb2, 0xc8, 0x21, 0x09, 0x10, 0xd4, 0x47, 0x37, 0x9b, 0x32, + 0x2d, 0x7b, 0xb2, 0x7b, 0x21, 0xbb, 0x5e, 0xfd, 0xde, 0xab, 0xaf, 0x57, 0x55, 0xbf, 0xf7, 0x0a, + 0x0a, 0x6c, 0x32, 0xa2, 0x41, 0x65, 0xe4, 0x7b, 0xcc, 0x43, 0xc8, 0xf2, 0xcc, 0x43, 0xea, 0x57, + 0x82, 0x63, 0xe2, 0x0f, 0x0f, 0x6d, 0x56, 0x39, 0xba, 0x57, 0xba, 0xc6, 0xec, 0x21, 0x0d, 0x18, + 0x19, 0x8e, 0x3e, 0x8e, 0xbe, 0x24, 0xbc, 0xf4, 0x8e, 0x35, 0xf6, 0x09, 0xb3, 0x3d, 0xf7, 0xe3, + 0xf0, 0x43, 0x55, 0x5c, 0x19, 0x78, 0x03, 0x4f, 0x7c, 0x7e, 0xcc, 0xbf, 0xa4, 0x54, 0x5f, 0x87, + 0xc5, 0xa7, 0xd4, 0x0f, 0x6c, 0xcf, 0x45, 0x57, 0x20, 0x63, 0xbb, 0x16, 0x7d, 0xbe, 0x9a, 0x28, + 0x27, 0x6e, 0xa5, 0xb1, 0x2c, 0xe8, 0x7f, 0x96, 0x80, 0x82, 0xe1, 0xba, 0x1e, 0x13, 0xb6, 0x02, + 0x84, 0x20, 0xed, 0x92, 0x21, 0x15, 0xa0, 0x3c, 0x16, 0xdf, 0xa8, 0x06, 0x59, 0x87, 0xec, 0x51, + 0x27, 0x58, 0x4d, 0x96, 0x53, 0xb7, 0x0a, 0x1b, 0x3f, 0xa8, 0xbc, 0xda, 0xe7, 0x4a, 0xcc, 0x48, + 0x65, 0x5b, 0xa0, 0x1b, 0x2e, 0xf3, 0x27, 0x58, 0xa9, 0x96, 0x7e, 0x08, 0x85, 0x98, 0x18, 0x69, + 0x90, 0x3a, 0xa4, 0x13, 0xd5, 0x0c, 0xff, 0xe4, 0xfd, 0x3b, 0x22, 0xce, 0x98, 0xae, 0x26, 0x85, + 0x4c, 0x16, 0xbe, 0x48, 0xde, 0x4f, 0xe8, 0x5f, 0x42, 0x1e, 0xd3, 0xc0, 0x1b, 0xfb, 0x26, 0x0d, + 0xd0, 0x47, 0x90, 0x77, 0x89, 0xeb, 0xf5, 0xcd, 0xd1, 0x38, 0x10, 0xea, 0xa9, 0x6a, 0xf1, 0xec, + 0xe5, 0x7a, 0xae, 0x45, 0x5c, 0xaf, 0xd6, 0xd9, 0x0d, 0x70, 0x8e, 0x57, 0xd7, 0x46, 0xe3, 0x00, + 0xbd, 0x0f, 0xc5, 0x21, 0x1d, 0x7a, 0xfe, 0xa4, 0xbf, 0x37, 0x61, 0x34, 0x10, 0x86, 0x53, 0xb8, + 0x20, 0x65, 0x55, 0x2e, 0xd2, 0xff, 0x38, 0x01, 0x57, 0x42, 0xdb, 0x98, 0xfe, 0xd6, 0xd8, 0xf6, + 0xe9, 0x90, 0xba, 0x2c, 0x40, 0x9f, 0x41, 0xd6, 0xb1, 0x87, 0x36, 0x93, 0x6d, 0x14, 0x36, 0xde, + 0x9b, 0x37, 0xe6, 0xa8, 0x57, 0x58, 0x81, 0x91, 0x01, 0x45, 0x9f, 0x06, 0xd4, 0x3f, 0x92, 0x33, + 0x21, 0x9a, 0x7c, 0xa3, 0xf2, 0x8c, 0x8a, 0xbe, 0x09, 0xb9, 0x8e, 0x43, 0xd8, 0xbe, 0xe7, 0x0f, + 0x91, 0x0e, 0x45, 0xe2, 0x9b, 0x07, 0x36, 0xa3, 0x26, 0x1b, 0xfb, 0xe1, 0xaa, 0xcc, 0xc8, 0xd0, + 0x55, 0x48, 0x7a, 0xb2, 0xa1, 0x7c, 0x35, 0x7b, 0xf6, 0x72, 0x3d, 0xd9, 0xee, 0xe2, 0xa4, 0x17, + 0xe8, 0x0f, 0xe0, 0x52, 0xc7, 0x19, 0x0f, 0x6c, 0xb7, 0x4e, 0x03, 0xd3, 0xb7, 0x47, 0xdc, 0x3a, + 0x5f, 0x5e, 0xee, 0x7c, 0xe1, 0xf2, 0xf2, 0xef, 0x68, 0xc9, 0x93, 0xd3, 0x25, 0xd7, 0xff, 0x30, + 0x09, 0x97, 0x1a, 0xee, 0xc0, 0x76, 0x69, 0x5c, 0xfb, 0x26, 0x2c, 0x53, 0x21, 0xec, 0x1f, 0x49, + 0xa7, 0x52, 0x76, 0x96, 0xa4, 0x34, 0xf4, 0xb4, 0xe6, 0x39, 0x7f, 0xb9, 0x37, 0x6f, 0xf8, 0xaf, + 0x58, 0x9f, 0xe7, 0x35, 0xa8, 0x01, 0x8b, 0x23, 0x31, 0x88, 0x60, 0x35, 0x25, 0x6c, 0xdd, 0x9c, + 0x67, 0xeb, 0x95, 0x71, 0x56, 0xd3, 0xdf, 0xbc, 0x5c, 0x5f, 0xc0, 0xa1, 0xee, 0x2f, 0xe3, 0x7c, + 0xff, 0x9e, 0x80, 0x95, 0x96, 0x67, 0xcd, 0xcc, 0x43, 0x09, 0x72, 0x07, 0x5e, 0xc0, 0x62, 0x1b, + 0x25, 0x2a, 0xa3, 0xfb, 0x90, 0x1b, 0xa9, 0xe5, 0x53, 0xab, 0x7f, 0x7d, 0x7e, 0x97, 0x25, 0x06, + 0x47, 0x68, 0xf4, 0x00, 0xf2, 0x7e, 0xe8, 0x13, 0xab, 0xa9, 0xb7, 0x71, 0x9c, 0x29, 0x1e, 0xfd, + 0x1a, 0x64, 0xe5, 0x22, 0xac, 0xa6, 0x85, 0xe6, 0xcd, 0xb7, 0x9a, 0x73, 0xac, 0x94, 0xf4, 0x9f, + 0x25, 0x40, 0xc3, 0x64, 0x9f, 0xed, 0xd0, 0xe1, 0x1e, 0xf5, 0xbb, 0x8c, 0xb0, 0x71, 0x80, 0xae, + 0x42, 0xd6, 0xa1, 0xc4, 0xa2, 0xbe, 0x18, 0x64, 0x0e, 0xab, 0x12, 0xda, 0xe5, 0x4e, 0x4e, 0xcc, + 0x03, 0xb2, 0x67, 0x3b, 0x36, 0x9b, 0x88, 0x61, 0x2e, 0xcf, 0x5f, 0xe5, 0xf3, 0x36, 0x2b, 0x38, + 0xa6, 0x88, 0x67, 0xcc, 0xa0, 0x55, 0x58, 0x1c, 0xd2, 0x20, 0x20, 0x03, 0x2a, 0x46, 0x9f, 0xc7, + 0x61, 0x51, 0x7f, 0x00, 0xc5, 0xb8, 0x1e, 0x2a, 0xc0, 0xe2, 0x6e, 0xeb, 0x49, 0xab, 0xfd, 0xac, + 0xa5, 0x2d, 0xa0, 0x15, 0x28, 0xec, 0xb6, 0x70, 0xc3, 0xa8, 0x6d, 0x19, 0xd5, 0xed, 0x86, 0x96, + 0x40, 0x4b, 0x90, 0x9f, 0x16, 0x93, 0xfa, 0x5f, 0x27, 0x00, 0xf8, 0x02, 0xaa, 0x41, 0x7d, 0x01, + 0x99, 0x80, 0x11, 0x26, 0x17, 0x6e, 0x79, 0xe3, 0x83, 0x79, 0xbd, 0x9e, 0xc2, 0x2b, 0xfc, 0x8f, + 0x62, 0xa9, 0x12, 0xef, 0x61, 0x72, 0xa6, 0x87, 0x7c, 0x0f, 0x11, 0xcb, 0xf2, 0x55, 0xc7, 0xc5, + 0xb7, 0xfe, 0x00, 0x32, 0x42, 0x7b, 0xb6, 0xbb, 0x39, 0x48, 0xd7, 0xf9, 0x57, 0x02, 0xe5, 0x21, + 0x83, 0x1b, 0x46, 0xfd, 0x4b, 0x2d, 0x89, 0x34, 0x28, 0xd6, 0x9b, 0xdd, 0x5a, 0xbb, 0xd5, 0x6a, + 0xd4, 0x7a, 0x8d, 0xba, 0x96, 0xd2, 0x6f, 0x42, 0xa6, 0x39, 0xe4, 0x96, 0xaf, 0x73, 0xaf, 0xd8, + 0xa7, 0x3e, 0x75, 0xcd, 0xd0, 0xd9, 0xa6, 0x02, 0xfd, 0xa7, 0x79, 0xc8, 0xec, 0x78, 0x63, 0x97, + 0xa1, 0x8d, 0xd8, 0xce, 0x5e, 0xde, 0x58, 0x9b, 0x37, 0x2c, 0x01, 0xac, 0xf4, 0x26, 0x23, 0xaa, + 0x76, 0xfe, 0x55, 0xc8, 0x4a, 0xff, 0x51, 0xc3, 0x51, 0x25, 0x2e, 0x67, 0xc4, 0x1f, 0x50, 0xa6, + 0xc6, 0xa3, 0x4a, 0xe8, 0x16, 0xe4, 0x7c, 0x4a, 0x2c, 0xcf, 0x75, 0x26, 0xc2, 0xcd, 0x72, 0xf2, + 0xe8, 0xc5, 0x94, 0x58, 0x6d, 0xd7, 0x99, 0xe0, 0xa8, 0x16, 0x6d, 0x41, 0x71, 0xcf, 0x76, 0xad, + 0xbe, 0x37, 0x92, 0xe7, 0x60, 0xe6, 0xf5, 0x4e, 0x29, 0x7b, 0x55, 0xb5, 0x5d, 0xab, 0x2d, 0xc1, + 0xb8, 0xb0, 0x37, 0x2d, 0xa0, 0x16, 0x2c, 0x1f, 0x79, 0xce, 0x78, 0x48, 0x23, 0x5b, 0x59, 0x61, + 0xeb, 0xc3, 0xd7, 0xdb, 0x7a, 0x2a, 0xf0, 0xa1, 0xb5, 0xa5, 0xa3, 0x78, 0x11, 0x3d, 0x81, 0x25, + 0x36, 0x1c, 0xed, 0x07, 0x91, 0xb9, 0x45, 0x61, 0xee, 0xfb, 0x17, 0x4c, 0x18, 0x87, 0x87, 0xd6, + 0x8a, 0x2c, 0x56, 0x2a, 0xfd, 0x7e, 0x0a, 0x0a, 0xb1, 0x9e, 0xa3, 0x2e, 0x14, 0x46, 0xbe, 0x37, + 0x22, 0x03, 0x71, 0x96, 0xab, 0xb5, 0xb8, 0xf7, 0x56, 0xa3, 0xae, 0x74, 0xa6, 0x8a, 0x38, 0x6e, + 0x45, 0x3f, 0x4d, 0x42, 0x21, 0x56, 0x89, 0x6e, 0x43, 0x0e, 0x77, 0x70, 0xf3, 0xa9, 0xd1, 0x6b, + 0x68, 0x0b, 0xa5, 0xeb, 0x27, 0xa7, 0xe5, 0x55, 0x61, 0x2d, 0x6e, 0xa0, 0xe3, 0xdb, 0x47, 0xdc, + 0xf5, 0x6e, 0xc1, 0x62, 0x08, 0x4d, 0x94, 0xde, 0x3d, 0x39, 0x2d, 0xbf, 0x73, 0x1e, 0x1a, 0x43, + 0xe2, 0xee, 0x96, 0x81, 0x1b, 0x75, 0x2d, 0x39, 0x1f, 0x89, 0xbb, 0x07, 0xc4, 0xa7, 0x16, 0xfa, + 0x3e, 0x64, 0x15, 0x30, 0x55, 0x2a, 0x9d, 0x9c, 0x96, 0xaf, 0x9e, 0x07, 0x4e, 0x71, 0xb8, 0xbb, + 0x6d, 0x3c, 0x6d, 0x68, 0xe9, 0xf9, 0x38, 0xdc, 0x75, 0xc8, 0x11, 0x45, 0x1f, 0x40, 0x46, 0xc2, + 0x32, 0xa5, 0x6b, 0x27, 0xa7, 0xe5, 0xef, 0xbd, 0x62, 0x8e, 0xa3, 0x4a, 0xab, 0x7f, 0xf4, 0xe7, + 0x6b, 0x0b, 0x7f, 0xfb, 0x17, 0x6b, 0xda, 0xf9, 0xea, 0xd2, 0x7f, 0x27, 0x60, 0x69, 0x66, 0xc9, + 0x91, 0x0e, 0x59, 0xd7, 0x33, 0xbd, 0x91, 0x3c, 0xe2, 0x73, 0x55, 0x38, 0x7b, 0xb9, 0x9e, 0x6d, + 0x79, 0x35, 0x6f, 0x34, 0xc1, 0xaa, 0x06, 0x3d, 0x39, 0x77, 0x49, 0x7d, 0xf2, 0x96, 0xfe, 0x34, + 0xf7, 0x9a, 0x7a, 0x08, 0x4b, 0x96, 0x6f, 0x1f, 0x51, 0xbf, 0x6f, 0x7a, 0xee, 0xbe, 0x3d, 0x50, + 0xc7, 0x77, 0x69, 0x9e, 0xcd, 0xba, 0x00, 0xe2, 0xa2, 0x54, 0xa8, 0x09, 0xfc, 0x2f, 0x71, 0x41, + 0x95, 0x9e, 0x42, 0x31, 0xee, 0xa1, 0xe8, 0x3d, 0x80, 0xc0, 0xfe, 0x6d, 0xaa, 0x38, 0x8f, 0x60, + 0x48, 0x38, 0xcf, 0x25, 0x82, 0xf1, 0xa0, 0x0f, 0x21, 0x3d, 0xf4, 0x2c, 0x69, 0x67, 0xa9, 0x7a, + 0x99, 0xdf, 0x93, 0xff, 0xfc, 0x72, 0xbd, 0xe0, 0x05, 0x95, 0x4d, 0xdb, 0xa1, 0x3b, 0x9e, 0x45, + 0xb1, 0x00, 0xe8, 0x47, 0x90, 0xe6, 0x47, 0x05, 0x7a, 0x17, 0xd2, 0xd5, 0x66, 0xab, 0xae, 0x2d, + 0x94, 0x2e, 0x9d, 0x9c, 0x96, 0x97, 0xc4, 0x94, 0xf0, 0x0a, 0xee, 0xbb, 0x68, 0x1d, 0xb2, 0x4f, + 0xdb, 0xdb, 0xbb, 0x3b, 0xdc, 0xbd, 0x2e, 0x9f, 0x9c, 0x96, 0x57, 0xa2, 0x6a, 0x39, 0x69, 0xe8, + 0x3d, 0xc8, 0xf4, 0x76, 0x3a, 0x9b, 0x5d, 0x2d, 0x59, 0x42, 0x27, 0xa7, 0xe5, 0xe5, 0xa8, 0x5e, + 0xf4, 0xb9, 0x74, 0x49, 0xad, 0x6a, 0x3e, 0x92, 0xeb, 0xff, 0x95, 0x84, 0x25, 0xcc, 0x39, 0xaf, + 0xcf, 0x3a, 0x9e, 0x63, 0x9b, 0x13, 0xd4, 0x81, 0xbc, 0xe9, 0xb9, 0x96, 0x1d, 0xdb, 0x53, 0x1b, + 0xaf, 0xb9, 0x18, 0xa7, 0x5a, 0x61, 0xa9, 0x16, 0x6a, 0xe2, 0xa9, 0x11, 0xb4, 0x01, 0x19, 0x8b, + 0x3a, 0x64, 0x72, 0xd1, 0x0d, 0x5d, 0x57, 0xfc, 0x1a, 0x4b, 0xa8, 0x60, 0x93, 0xe4, 0x79, 0x9f, + 0x30, 0x46, 0x87, 0x23, 0x26, 0x6f, 0xe8, 0x34, 0x2e, 0x0c, 0xc9, 0x73, 0x43, 0x89, 0xd0, 0xa7, + 0x90, 0x3d, 0xb6, 0x5d, 0xcb, 0x3b, 0x56, 0x97, 0xf0, 0xc5, 0x76, 0x15, 0x56, 0x3f, 0xe1, 0x77, + 0xef, 0xb9, 0xce, 0xf2, 0x59, 0x6f, 0xb5, 0x5b, 0x8d, 0x70, 0xd6, 0x55, 0x7d, 0xdb, 0x6d, 0x79, + 0x2e, 0xdf, 0x31, 0xd0, 0x6e, 0xf5, 0x37, 0x8d, 0xe6, 0xf6, 0x2e, 0xe6, 0x33, 0x7f, 0xe5, 0xe4, + 0xb4, 0xac, 0x45, 0x90, 0x4d, 0x62, 0x3b, 0x9c, 0x18, 0x5e, 0x83, 0x94, 0xd1, 0xfa, 0x52, 0x4b, + 0x96, 0xb4, 0x93, 0xd3, 0x72, 0x31, 0xaa, 0x36, 0xdc, 0xc9, 0x74, 0x33, 0x9d, 0x6f, 0x57, 0xff, + 0xd7, 0x24, 0x14, 0x77, 0x47, 0x16, 0x61, 0x54, 0x7a, 0x26, 0x2a, 0x43, 0x61, 0x44, 0x7c, 0xe2, + 0x38, 0xd4, 0xb1, 0x83, 0xa1, 0x0a, 0x1e, 0xe2, 0x22, 0x74, 0xff, 0x3b, 0x4c, 0xa6, 0x22, 0x66, + 0x6a, 0x4a, 0x77, 0x61, 0x79, 0x5f, 0x76, 0xb6, 0x4f, 0x4c, 0xb1, 0xba, 0x29, 0xb1, 0xba, 0x95, + 0x79, 0x26, 0xe2, 0xbd, 0xaa, 0xa8, 0x31, 0x1a, 0x42, 0x0b, 0x2f, 0xed, 0xc7, 0x8b, 0xe8, 0x73, + 0x58, 0x1c, 0x7a, 0xae, 0xcd, 0x3c, 0xff, 0xad, 0xd6, 0x21, 0x04, 0xa3, 0xdb, 0x70, 0x89, 0xaf, + 0x70, 0xd8, 0x25, 0x51, 0x2d, 0x6e, 0xae, 0x24, 0x5e, 0x19, 0x92, 0xe7, 0xaa, 0x4d, 0xcc, 0xc5, + 0xfa, 0xe7, 0xb0, 0x34, 0xd3, 0x07, 0x7e, 0x9b, 0x77, 0x8c, 0xdd, 0x6e, 0x43, 0x5b, 0x40, 0x45, + 0xc8, 0xd5, 0xda, 0xad, 0x5e, 0xb3, 0xb5, 0xcb, 0xe9, 0x48, 0x11, 0x72, 0xb8, 0xbd, 0xbd, 0x5d, + 0x35, 0x6a, 0x4f, 0xb4, 0xa4, 0xfe, 0x8b, 0x68, 0x7e, 0x15, 0x1f, 0xa9, 0xce, 0xf2, 0x91, 0x3b, + 0xaf, 0x1f, 0xba, 0x62, 0x24, 0xd3, 0x42, 0xc4, 0x4b, 0xfe, 0x3f, 0x80, 0x58, 0x46, 0x6a, 0xf5, + 0x09, 0xbb, 0x28, 0xe6, 0xe8, 0x85, 0xd1, 0x24, 0xce, 0x2b, 0x05, 0x83, 0xa1, 0x47, 0x50, 0x34, + 0xbd, 0xe1, 0xc8, 0xa1, 0x4a, 0x3f, 0xf5, 0x36, 0xfa, 0x85, 0x48, 0xc5, 0x60, 0x71, 0x5e, 0x94, + 0x9e, 0x65, 0x6e, 0x7f, 0x90, 0x80, 0x42, 0xac, 0xc3, 0xb3, 0x54, 0xa8, 0x08, 0xb9, 0xdd, 0x4e, + 0xdd, 0xe8, 0x35, 0x5b, 0x8f, 0xb5, 0x04, 0x02, 0xc8, 0x8a, 0x09, 0xac, 0x6b, 0x49, 0x4e, 0xe1, + 0x6a, 0xed, 0x9d, 0xce, 0x76, 0x43, 0x90, 0x21, 0x74, 0x05, 0xb4, 0x70, 0x0a, 0xfb, 0xdd, 0x9e, + 0x81, 0xb9, 0x34, 0x8d, 0x2e, 0xc3, 0x4a, 0x24, 0x55, 0x9a, 0x19, 0x74, 0x15, 0x50, 0x24, 0x9c, + 0x9a, 0xc8, 0xea, 0xbf, 0x0b, 0x2b, 0x35, 0xcf, 0x65, 0xc4, 0x76, 0x23, 0x7a, 0xbb, 0xc1, 0xc7, + 0xad, 0x44, 0x7d, 0xdb, 0x92, 0xa7, 0x6d, 0x75, 0xe5, 0xec, 0xe5, 0x7a, 0x21, 0x82, 0x36, 0xeb, + 0x7c, 0xa4, 0x61, 0xc1, 0xe2, 0x7b, 0x6a, 0x64, 0x5b, 0x62, 0x8a, 0x33, 0xd5, 0xc5, 0xb3, 0x97, + 0xeb, 0xa9, 0x4e, 0xb3, 0x8e, 0xb9, 0x0c, 0xbd, 0x0b, 0x79, 0xfa, 0xdc, 0x66, 0x7d, 0x93, 0x9f, + 0xae, 0x7c, 0x0e, 0x33, 0x38, 0xc7, 0x05, 0x35, 0x7e, 0x98, 0x56, 0x01, 0x3a, 0x9e, 0xcf, 0x54, + 0xcb, 0x9f, 0x42, 0x66, 0xe4, 0xf9, 0x22, 0xb6, 0xe4, 0x57, 0xcf, 0x5c, 0xb2, 0xc6, 0xe1, 0xd2, + 0xd9, 0xb1, 0x04, 0xeb, 0x7f, 0x97, 0x04, 0xe8, 0x91, 0xe0, 0x50, 0x19, 0x79, 0x00, 0xf9, 0x28, + 0x39, 0x70, 0x51, 0x90, 0x1a, 0x5b, 0xf3, 0x08, 0x8f, 0x3e, 0x09, 0xbd, 0x4e, 0x72, 0xf7, 0xf9, + 0x8a, 0xaa, 0xad, 0x79, 0xf4, 0x77, 0x96, 0xa0, 0xf3, 0xfb, 0x8a, 0xfa, 0xbe, 0x5a, 0x7c, 0xfe, + 0x89, 0x6a, 0xe2, 0xcc, 0x96, 0xf3, 0xa6, 0xd8, 0xdf, 0x8d, 0x79, 0x8d, 0x9c, 0x5b, 0x94, 0xad, + 0x05, 0x3c, 0xd5, 0x43, 0x0f, 0xa1, 0xc0, 0x87, 0xde, 0x0f, 0x44, 0x9d, 0x22, 0x7e, 0xaf, 0x9d, + 0x2d, 0x69, 0x01, 0xc3, 0x28, 0xfa, 0xae, 0x6a, 0xb0, 0xec, 0x8f, 0x5d, 0x3e, 0x6c, 0x65, 0x43, + 0xb7, 0xe1, 0x9d, 0x16, 0x65, 0xc7, 0x9e, 0x7f, 0x68, 0x30, 0x46, 0xcc, 0x03, 0x1e, 0xed, 0xab, + 0x93, 0x6e, 0xca, 0x7a, 0x13, 0x33, 0xac, 0x77, 0x15, 0x16, 0x89, 0x63, 0x93, 0x80, 0x4a, 0xaa, + 0x90, 0xc7, 0x61, 0x91, 0x73, 0x73, 0xce, 0xf4, 0x69, 0x10, 0x50, 0x19, 0x9f, 0xe6, 0xf1, 0x54, + 0xa0, 0xff, 0x63, 0x12, 0xa0, 0xd9, 0x31, 0x76, 0x94, 0xf9, 0x3a, 0x64, 0xf7, 0xc9, 0xd0, 0x76, + 0x26, 0x17, 0xed, 0xf4, 0x29, 0xbe, 0x62, 0x48, 0x43, 0x9b, 0x42, 0x07, 0x2b, 0x5d, 0x41, 0xd9, + 0xc7, 0x7b, 0x2e, 0x65, 0x11, 0x65, 0x17, 0x25, 0xce, 0x0f, 0x7c, 0xe2, 0x46, 0x2b, 0x23, 0x0b, + 0xbc, 0xeb, 0x03, 0xc2, 0xe8, 0x31, 0x99, 0x84, 0x1b, 0x53, 0x15, 0xd1, 0x16, 0xa7, 0xf2, 0x01, + 0xf5, 0x8f, 0xa8, 0xb5, 0x9a, 0x11, 0x5e, 0xf8, 0xa6, 0xfe, 0x60, 0x05, 0x97, 0xcc, 0x27, 0xd2, + 0x2e, 0x3d, 0x10, 0xd7, 0xf5, 0xb4, 0xea, 0x3b, 0x45, 0xd7, 0x77, 0x61, 0x69, 0x66, 0x9c, 0xaf, + 0xc4, 0x4a, 0xcd, 0xce, 0xd3, 0x4f, 0xb5, 0xb4, 0xfa, 0xfa, 0x5c, 0xcb, 0xea, 0x7f, 0x95, 0x92, + 0x5b, 0x49, 0xcd, 0xea, 0xfc, 0x7c, 0x55, 0x4e, 0x64, 0xbf, 0x4c, 0xcf, 0x51, 0xfe, 0xfd, 0xe1, + 0xc5, 0x3b, 0x8c, 0x73, 0x6f, 0x01, 0xc7, 0x91, 0x22, 0x5a, 0x87, 0x82, 0x5c, 0xff, 0x3e, 0xf7, + 0x27, 0x31, 0xad, 0x4b, 0x18, 0xa4, 0x88, 0x6b, 0xa2, 0x9b, 0xb0, 0x3c, 0x1a, 0xef, 0x39, 0x76, + 0x70, 0x40, 0x2d, 0x89, 0x49, 0x0b, 0xcc, 0x52, 0x24, 0x15, 0xb0, 0x1d, 0x28, 0x2a, 0x41, 0x5f, + 0xf0, 0xae, 0x8c, 0xe8, 0xd0, 0xed, 0x37, 0x75, 0x48, 0xaa, 0x08, 0x3a, 0x56, 0x18, 0x4d, 0x0b, + 0x7a, 0x1d, 0x72, 0x61, 0x67, 0xd1, 0x2a, 0xa4, 0x7a, 0xb5, 0x8e, 0xb6, 0x50, 0x5a, 0x39, 0x39, + 0x2d, 0x17, 0x42, 0x71, 0xaf, 0xd6, 0xe1, 0x35, 0xbb, 0xf5, 0x8e, 0x96, 0x98, 0xad, 0xd9, 0xad, + 0x77, 0x4a, 0x69, 0x7e, 0xf3, 0xeb, 0xfb, 0x50, 0x88, 0xb5, 0x80, 0x6e, 0xc0, 0x62, 0xb3, 0xf5, + 0x18, 0x37, 0xba, 0x5d, 0x6d, 0xa1, 0x74, 0xf5, 0xe4, 0xb4, 0x8c, 0x62, 0xb5, 0x4d, 0x77, 0xc0, + 0xd7, 0x07, 0xbd, 0x07, 0xe9, 0xad, 0x76, 0xb7, 0x17, 0x12, 0xbd, 0x18, 0x62, 0xcb, 0x0b, 0x58, + 0xe9, 0xb2, 0xa2, 0x14, 0x71, 0xc3, 0xfa, 0x9f, 0x24, 0x20, 0x2b, 0xf9, 0xee, 0xdc, 0x85, 0x32, + 0x60, 0x31, 0x8c, 0xc2, 0x24, 0x09, 0xff, 0xf0, 0xf5, 0x84, 0xb9, 0xa2, 0xf8, 0xad, 0x74, 0xbf, + 0x50, 0xaf, 0xf4, 0x05, 0x14, 0xe3, 0x15, 0xdf, 0xc9, 0xf9, 0x7e, 0x07, 0x0a, 0xdc, 0xbf, 0x43, + 0xe2, 0xbc, 0x01, 0x59, 0xc9, 0xc9, 0xd5, 0x69, 0x7a, 0x11, 0x7b, 0x57, 0x48, 0x74, 0x1f, 0x16, + 0x25, 0xe3, 0x0f, 0xf3, 0x53, 0x6b, 0x17, 0xef, 0x22, 0x1c, 0xc2, 0xf5, 0x87, 0x90, 0xee, 0x50, + 0xea, 0xf3, 0xb9, 0x77, 0x3d, 0x8b, 0x4e, 0x2f, 0x20, 0x15, 0xac, 0x58, 0xb4, 0x59, 0xe7, 0xc1, + 0x8a, 0x45, 0x9b, 0x56, 0x94, 0x5e, 0x48, 0xc6, 0xd2, 0x0b, 0x3d, 0x28, 0x3e, 0xa3, 0xf6, 0xe0, + 0x80, 0x51, 0x4b, 0x18, 0xba, 0x03, 0xe9, 0x11, 0x8d, 0x3a, 0xbf, 0x3a, 0xd7, 0xc1, 0x28, 0xf5, + 0xb1, 0x40, 0xf1, 0x73, 0xe4, 0x58, 0x68, 0xab, 0xac, 0xa8, 0x2a, 0xe9, 0xff, 0x90, 0x84, 0xe5, + 0x66, 0x10, 0x8c, 0x89, 0x6b, 0x86, 0x0c, 0xe5, 0x47, 0xb3, 0x0c, 0xe5, 0xd6, 0xdc, 0x11, 0xce, + 0xa8, 0xcc, 0x66, 0x4d, 0xd4, 0xe5, 0x90, 0x8c, 0x2e, 0x07, 0xfd, 0x3f, 0x12, 0x61, 0x6a, 0xe4, + 0x66, 0x6c, 0xbb, 0x97, 0x56, 0x4f, 0x4e, 0xcb, 0x57, 0xe2, 0x96, 0xe8, 0xae, 0x7b, 0xe8, 0x7a, + 0xc7, 0x2e, 0x7a, 0x1f, 0x32, 0xb8, 0xd1, 0x6a, 0x3c, 0xd3, 0x12, 0xd2, 0x3d, 0x67, 0x40, 0x98, + 0xba, 0xf4, 0x98, 0x5b, 0xea, 0x34, 0x5a, 0x75, 0xce, 0x25, 0x92, 0x73, 0x2c, 0x75, 0xa8, 0x6b, + 0xd9, 0xee, 0x00, 0xdd, 0x80, 0x6c, 0xb3, 0xdb, 0xdd, 0x15, 0xc1, 0xeb, 0x3b, 0x27, 0xa7, 0xe5, + 0xcb, 0x33, 0x28, 0x5e, 0xa0, 0x16, 0x07, 0x71, 0x72, 0xcd, 0x59, 0xc6, 0x1c, 0x10, 0xe7, 0x7d, + 0x12, 0x84, 0xdb, 0x3d, 0x1e, 0x59, 0x67, 0xe6, 0x80, 0xb0, 0xc7, 0x7f, 0xd5, 0x76, 0xfb, 0x97, + 0x24, 0x68, 0x86, 0x69, 0xd2, 0x11, 0xe3, 0xf5, 0x2a, 0xaa, 0xe9, 0x41, 0x6e, 0xc4, 0xbf, 0x6c, + 0x1a, 0xf2, 0x80, 0xfb, 0x73, 0xf3, 0xea, 0xe7, 0xf4, 0x2a, 0xd8, 0x73, 0xa8, 0x61, 0x0d, 0xed, + 0x20, 0xe0, 0xd1, 0xbb, 0x90, 0xe1, 0xc8, 0x52, 0xe9, 0xe7, 0x09, 0xb8, 0x3c, 0x07, 0x81, 0xee, + 0x42, 0xda, 0xf7, 0x9c, 0x70, 0x0d, 0xaf, 0xbf, 0x2e, 0xeb, 0xc5, 0x55, 0xb1, 0x40, 0xa2, 0x35, + 0x00, 0x32, 0x66, 0x1e, 0x11, 0xed, 0x8b, 0xd5, 0xcb, 0xe1, 0x98, 0x04, 0x3d, 0x83, 0x6c, 0x40, + 0x4d, 0x9f, 0x86, 0x84, 0xf1, 0xe1, 0xff, 0xb5, 0xf7, 0x95, 0xae, 0x30, 0x83, 0x95, 0xb9, 0x52, + 0x05, 0xb2, 0x52, 0xc2, 0xdd, 0xde, 0x22, 0x8c, 0x88, 0x4e, 0x17, 0xb1, 0xf8, 0xe6, 0xde, 0x44, + 0x9c, 0x41, 0xe8, 0x4d, 0xc4, 0x19, 0xe8, 0x7f, 0x9a, 0x04, 0x68, 0x3c, 0x67, 0xd4, 0x77, 0x89, + 0x53, 0x33, 0x50, 0x23, 0x76, 0xfa, 0xcb, 0xd1, 0x7e, 0x34, 0x37, 0x17, 0x1a, 0x69, 0x54, 0x6a, + 0xc6, 0x9c, 0xf3, 0xff, 0x1a, 0xa4, 0xc6, 0xbe, 0xa3, 0xf2, 0xea, 0x82, 0xe9, 0xed, 0xe2, 0x6d, + 0xcc, 0x65, 0xa8, 0x31, 0x3d, 0xb6, 0x52, 0xaf, 0x7f, 0x10, 0x89, 0x35, 0xf0, 0xab, 0x3f, 0xba, + 0xee, 0x00, 0x4c, 0x7b, 0x8d, 0xd6, 0x20, 0x53, 0xdb, 0xec, 0x76, 0xb7, 0xb5, 0x05, 0x79, 0x36, + 0x4f, 0xab, 0x84, 0x58, 0xff, 0xcb, 0x04, 0xe4, 0x6a, 0x86, 0xba, 0x31, 0x37, 0x41, 0x13, 0x07, + 0x8e, 0x49, 0x7d, 0xd6, 0xa7, 0xcf, 0x47, 0xb6, 0x3f, 0x51, 0x67, 0xc6, 0xc5, 0x61, 0xd2, 0x32, + 0xd7, 0xaa, 0x51, 0x9f, 0x35, 0x84, 0x0e, 0xc2, 0x50, 0xa4, 0x6a, 0x88, 0x7d, 0x93, 0x84, 0x27, + 0xf8, 0xda, 0xc5, 0x53, 0x21, 0xe9, 0xf5, 0xb4, 0x1c, 0xe0, 0x42, 0x68, 0xa4, 0x46, 0x02, 0xfd, + 0x29, 0x5c, 0x6e, 0xfb, 0xe6, 0x01, 0x0d, 0x98, 0x6c, 0x54, 0x75, 0xf9, 0x21, 0x5c, 0x67, 0x24, + 0x38, 0xec, 0x1f, 0xd8, 0x01, 0xf3, 0xfc, 0x49, 0xdf, 0xa7, 0x8c, 0xba, 0xbc, 0xbe, 0x2f, 0x9e, + 0x5d, 0x54, 0x92, 0xe3, 0x1a, 0xc7, 0x6c, 0x49, 0x08, 0x0e, 0x11, 0xdb, 0x1c, 0xa0, 0x37, 0xa1, + 0xc8, 0xd9, 0x6c, 0x9d, 0xee, 0x93, 0xb1, 0xc3, 0x02, 0xf4, 0x43, 0x00, 0xc7, 0x1b, 0xf4, 0xdf, + 0xfa, 0xb8, 0xcf, 0x3b, 0xde, 0x40, 0x7e, 0xea, 0xbf, 0x01, 0x5a, 0xdd, 0x0e, 0x46, 0x84, 0x99, + 0x07, 0x61, 0xf6, 0x06, 0x3d, 0x06, 0xed, 0x80, 0x12, 0x9f, 0xed, 0x51, 0xc2, 0xfa, 0x23, 0xea, + 0xdb, 0x9e, 0xf5, 0x56, 0x53, 0xba, 0x12, 0x69, 0x75, 0x84, 0x92, 0xfe, 0x9f, 0x09, 0x00, 0x4c, + 0xf6, 0x43, 0x72, 0xf3, 0x03, 0xb8, 0x14, 0xb8, 0x64, 0x14, 0x1c, 0x78, 0xac, 0x6f, 0xbb, 0x8c, + 0xfa, 0x47, 0xc4, 0x51, 0x11, 0xb8, 0x16, 0x56, 0x34, 0x95, 0x1c, 0xdd, 0x01, 0x74, 0x48, 0xe9, + 0xa8, 0xef, 0x39, 0x56, 0x3f, 0xac, 0x94, 0xef, 0x42, 0x69, 0xac, 0xf1, 0x9a, 0xb6, 0x63, 0x75, + 0x43, 0x39, 0xaa, 0xc2, 0x1a, 0x9f, 0x01, 0xea, 0x32, 0xdf, 0xa6, 0x41, 0x7f, 0xdf, 0xf3, 0xfb, + 0x81, 0xe3, 0x1d, 0xf7, 0xf7, 0x3d, 0xc7, 0xf1, 0x8e, 0xa9, 0x1f, 0xe6, 0x37, 0x4a, 0x8e, 0x37, + 0x68, 0x48, 0xd0, 0xa6, 0xe7, 0x77, 0x1d, 0xef, 0x78, 0x33, 0x44, 0x70, 0x06, 0x34, 0x1d, 0x36, + 0xb3, 0xcd, 0xc3, 0x90, 0x01, 0x45, 0xd2, 0x9e, 0x6d, 0x1e, 0xa2, 0x1b, 0xb0, 0x44, 0x1d, 0x2a, + 0xa2, 0x64, 0x89, 0xca, 0x08, 0x54, 0x31, 0x14, 0x72, 0x90, 0xfe, 0x08, 0xb4, 0x86, 0x6b, 0xfa, + 0x93, 0x51, 0x6c, 0xd9, 0xef, 0x00, 0xe2, 0xe7, 0x4d, 0xdf, 0xf1, 0xcc, 0xc3, 0xfe, 0x90, 0xb8, + 0x64, 0xc0, 0xfb, 0x25, 0xdf, 0x22, 0x34, 0x5e, 0xb3, 0xed, 0x99, 0x87, 0x3b, 0x4a, 0xae, 0xff, + 0x3f, 0xc8, 0x77, 0x1c, 0x62, 0x8a, 0xf7, 0x3b, 0x54, 0x06, 0x1e, 0xb6, 0x71, 0x37, 0xb2, 0x5d, + 0x15, 0x67, 0xe5, 0x71, 0x5c, 0xa4, 0xff, 0x08, 0xe0, 0xc7, 0x9e, 0xed, 0xf6, 0xbc, 0x43, 0xea, + 0x8a, 0xa7, 0x0e, 0x1e, 0x13, 0x28, 0x67, 0xc8, 0x63, 0x55, 0x12, 0x21, 0x8f, 0x6c, 0x20, 0xca, + 0xf8, 0xcb, 0xa2, 0xfe, 0x4d, 0x02, 0xb2, 0xd8, 0xf3, 0x58, 0xcd, 0x40, 0x65, 0xc8, 0x9a, 0xa4, + 0x1f, 0xee, 0xdd, 0x62, 0x35, 0x7f, 0xf6, 0x72, 0x3d, 0x53, 0x33, 0x9e, 0xd0, 0x09, 0xce, 0x98, + 0xe4, 0x09, 0x9d, 0xf0, 0x4b, 0xde, 0x24, 0x62, 0xc7, 0x09, 0x33, 0x45, 0x79, 0xc9, 0xd7, 0x0c, + 0xbe, 0x9d, 0x70, 0xd6, 0x24, 0xfc, 0x1f, 0xdd, 0x85, 0xa2, 0x02, 0xf5, 0x0f, 0x48, 0x70, 0x20, + 0x99, 0x7c, 0x75, 0xf9, 0xec, 0xe5, 0x3a, 0x48, 0xe4, 0x16, 0x09, 0x0e, 0x30, 0x48, 0x34, 0xff, + 0x46, 0x0d, 0x28, 0x7c, 0xe5, 0xd9, 0x6e, 0x9f, 0x89, 0x41, 0xa8, 0x64, 0xc7, 0xdc, 0x1d, 0x38, + 0x1d, 0xaa, 0xca, 0xc0, 0xc0, 0x57, 0x91, 0x44, 0xff, 0xa7, 0x04, 0x14, 0xb8, 0x4d, 0x7b, 0xdf, + 0x36, 0xf9, 0xa5, 0xfc, 0xdd, 0xef, 0x8a, 0x6b, 0x90, 0x32, 0x03, 0x5f, 0x8d, 0x4d, 0x1c, 0x96, + 0xb5, 0x2e, 0xc6, 0x5c, 0x86, 0x1e, 0x41, 0x56, 0x85, 0x6f, 0xf2, 0x9a, 0xd0, 0xdf, 0x4c, 0x1f, + 0x54, 0x17, 0x95, 0x9e, 0x58, 0xcb, 0x69, 0xef, 0xc4, 0x28, 0x8b, 0x38, 0x2e, 0x42, 0x57, 0x21, + 0x69, 0xba, 0xc2, 0xad, 0xd4, 0x13, 0x68, 0xad, 0x85, 0x93, 0xa6, 0xab, 0xff, 0x7d, 0x02, 0x96, + 0xa6, 0x5e, 0xc5, 0x17, 0xe2, 0x3a, 0xe4, 0x83, 0xf1, 0x5e, 0x30, 0x09, 0x18, 0x1d, 0x86, 0xaf, + 0x29, 0x91, 0x00, 0x35, 0x21, 0x4f, 0x9c, 0x81, 0xe7, 0xdb, 0xec, 0x60, 0xa8, 0x22, 0x87, 0xf9, + 0x47, 0x7b, 0xdc, 0x66, 0xc5, 0x08, 0x55, 0xf0, 0x54, 0x3b, 0x3c, 0xcc, 0x53, 0xa2, 0xb3, 0xe2, + 0x30, 0x7f, 0x1f, 0x8a, 0x0e, 0x19, 0x8a, 0x78, 0x96, 0x07, 0xa4, 0x62, 0x1c, 0x69, 0x5c, 0x50, + 0x32, 0x1e, 0xa5, 0xeb, 0x3a, 0xe4, 0x23, 0x63, 0x68, 0x05, 0x0a, 0x46, 0xa3, 0xdb, 0xbf, 0xb7, + 0x71, 0xbf, 0xff, 0xb8, 0xb6, 0xa3, 0x2d, 0x28, 0x2e, 0xf1, 0x37, 0x09, 0x58, 0x52, 0x3e, 0xaf, + 0xf8, 0xd9, 0x0d, 0x58, 0xf4, 0xc9, 0x3e, 0x0b, 0x19, 0x64, 0x5a, 0x3a, 0x17, 0x3f, 0x46, 0x38, + 0x83, 0xe4, 0x55, 0xf3, 0x19, 0x64, 0xec, 0x7d, 0x2f, 0x75, 0xe1, 0xfb, 0x5e, 0xfa, 0x57, 0xf2, + 0xbe, 0xa7, 0xff, 0x24, 0x09, 0x2b, 0xea, 0xaa, 0x0f, 0xdf, 0xaf, 0xd0, 0x47, 0x90, 0x97, 0xb7, + 0xfe, 0x94, 0xff, 0x8a, 0x27, 0x25, 0x89, 0x6b, 0xd6, 0x71, 0x4e, 0x56, 0x37, 0x2d, 0x1e, 0x90, + 0x29, 0x68, 0xec, 0xb5, 0x1a, 0xa4, 0xa8, 0xc5, 0xa3, 0x89, 0x3a, 0xa4, 0xf7, 0x6d, 0x87, 0x2a, + 0x3f, 0x9b, 0x9b, 0x43, 0x3c, 0xd7, 0xbc, 0x48, 0x79, 0xf7, 0x44, 0x48, 0xb7, 0xb5, 0x80, 0x85, + 0x76, 0xe9, 0xf7, 0x00, 0xa6, 0xd2, 0xb9, 0x51, 0x0b, 0x67, 0x06, 0x2a, 0x07, 0x14, 0x32, 0x83, + 0x66, 0x1d, 0x73, 0x19, 0xaf, 0x1a, 0xd8, 0x96, 0xda, 0xb9, 0xa2, 0xea, 0x31, 0xaf, 0x1a, 0xd8, + 0x56, 0x94, 0x77, 0x4f, 0xbf, 0x21, 0xef, 0x5e, 0xcd, 0x85, 0x69, 0x08, 0xbd, 0x0d, 0x57, 0xab, + 0x0e, 0x31, 0x0f, 0x1d, 0x3b, 0x60, 0xd4, 0x8a, 0xef, 0xd0, 0xcf, 0x20, 0x3b, 0x73, 0x73, 0xbf, + 0x21, 0xf1, 0xa3, 0xc0, 0xfa, 0x4f, 0x12, 0x50, 0xdc, 0xa2, 0xc4, 0x61, 0x07, 0xd3, 0xe8, 0x99, + 0xd1, 0x80, 0xa9, 0xf3, 0x51, 0x7c, 0xa3, 0xfb, 0x90, 0x8b, 0xee, 0x9a, 0xb7, 0x49, 0x8f, 0x47, + 0x68, 0xf4, 0x39, 0x2c, 0x72, 0xcf, 0xf6, 0xc6, 0x21, 0x25, 0x7c, 0x43, 0xde, 0x55, 0x81, 0xf9, + 0x21, 0xeb, 0x53, 0x71, 0xc5, 0x88, 0xd9, 0xc9, 0xe0, 0xb0, 0xa8, 0xff, 0x4f, 0x02, 0xae, 0xec, + 0x90, 0xc9, 0x1e, 0x55, 0x3b, 0x8e, 0x5a, 0x98, 0x9a, 0x9e, 0x6f, 0xa1, 0x4e, 0x7c, 0xa7, 0x5e, + 0xf0, 0x24, 0x30, 0x4f, 0x79, 0xfe, 0x86, 0x0d, 0xb9, 0x66, 0x32, 0xc6, 0x35, 0xaf, 0x40, 0xc6, + 0xf5, 0x5c, 0x93, 0xaa, 0x6d, 0x2c, 0x0b, 0xba, 0x1d, 0xdf, 0xa5, 0xa5, 0x28, 0x4f, 0x2f, 0xb2, + 0xec, 0x2d, 0x8f, 0x45, 0xad, 0xa1, 0x47, 0x50, 0xea, 0x36, 0x6a, 0xb8, 0xd1, 0xab, 0xb6, 0x7f, + 0xbd, 0xdf, 0x35, 0xb6, 0xbb, 0xc6, 0xc6, 0xdd, 0x7e, 0xa7, 0xbd, 0xfd, 0xe5, 0xbd, 0x4f, 0xee, + 0x7e, 0xa6, 0x25, 0x4a, 0xe5, 0x93, 0xd3, 0xf2, 0xf5, 0x96, 0x51, 0xdb, 0x96, 0x6e, 0xb9, 0xe7, + 0x3d, 0xef, 0x12, 0x27, 0x20, 0x1b, 0x77, 0x3b, 0x9e, 0x33, 0xe1, 0x98, 0xdb, 0xbf, 0x48, 0x41, + 0x3e, 0x4a, 0xc3, 0x71, 0xef, 0xe2, 0x31, 0x90, 0x6a, 0x2a, 0x92, 0xb7, 0xe8, 0x31, 0x7a, 0x7f, + 0x1a, 0xfd, 0x3c, 0x92, 0xcf, 0x01, 0x51, 0x75, 0x18, 0xf9, 0x7c, 0x00, 0x39, 0xa3, 0xdb, 0x6d, + 0x3e, 0x6e, 0x35, 0xea, 0xda, 0xd7, 0x89, 0xd2, 0xf7, 0x4e, 0x4e, 0xcb, 0x97, 0x22, 0x90, 0x11, + 0x04, 0xf6, 0xc0, 0xa5, 0x96, 0x40, 0xd5, 0x6a, 0x8d, 0x4e, 0xaf, 0x51, 0xd7, 0x5e, 0x24, 0xcf, + 0xa3, 0x04, 0x9b, 0x17, 0x4f, 0x7b, 0xf9, 0x0e, 0x6e, 0x74, 0x0c, 0xcc, 0x1b, 0xfc, 0x3a, 0x29, + 0x83, 0xb2, 0x69, 0x8b, 0x3e, 0x1d, 0x11, 0x9f, 0xb7, 0xb9, 0x16, 0x3e, 0x71, 0xbf, 0x48, 0xc9, + 0xe7, 0x9f, 0x69, 0x4e, 0x91, 0x12, 0x6b, 0xc2, 0x5b, 0x13, 0xf9, 0x5c, 0x61, 0x26, 0x75, 0xae, + 0xb5, 0x2e, 0x23, 0x3e, 0xe3, 0x56, 0x74, 0x58, 0xc4, 0xbb, 0xad, 0x16, 0x07, 0xbd, 0x48, 0x9f, + 0x1b, 0x1d, 0x1e, 0xbb, 0x2e, 0xc7, 0xdc, 0x84, 0x5c, 0x98, 0xee, 0xd5, 0xbe, 0x4e, 0x9f, 0xeb, + 0x50, 0x2d, 0xcc, 0x55, 0x8b, 0x06, 0xb7, 0x76, 0x7b, 0xe2, 0x05, 0xfe, 0x45, 0xe6, 0x7c, 0x83, + 0x07, 0x63, 0x66, 0xf1, 0x70, 0xb3, 0x1c, 0xc5, 0x7f, 0x5f, 0x67, 0x24, 0xa3, 0x8e, 0x30, 0x2a, + 0xf8, 0xfb, 0x00, 0x72, 0xb8, 0xf1, 0x63, 0xf9, 0x58, 0xff, 0x22, 0x7b, 0xce, 0x0e, 0xa6, 0x5f, + 0x51, 0x53, 0xb5, 0xd6, 0xc6, 0x9d, 0x2d, 0x43, 0x4c, 0xf9, 0x79, 0x54, 0xdb, 0x1f, 0x1d, 0x10, + 0x97, 0x5a, 0xd3, 0x37, 0xb0, 0xa8, 0xea, 0xf6, 0x6f, 0x42, 0x2e, 0xbc, 0x61, 0xd1, 0x1a, 0x64, + 0x9f, 0xb5, 0xf1, 0x93, 0x06, 0xd6, 0x16, 0xe4, 0x1c, 0x86, 0x35, 0xcf, 0x24, 0x45, 0x29, 0xc3, + 0xe2, 0x8e, 0xd1, 0x32, 0x1e, 0x37, 0x70, 0x98, 0x9a, 0x09, 0x01, 0xea, 0x9a, 0x28, 0x69, 0xaa, + 0x81, 0xc8, 0x66, 0xf5, 0xfa, 0x37, 0xdf, 0xae, 0x2d, 0xfc, 0xec, 0xdb, 0xb5, 0x85, 0x9f, 0x7f, + 0xbb, 0x96, 0x78, 0x71, 0xb6, 0x96, 0xf8, 0xe6, 0x6c, 0x2d, 0xf1, 0xd3, 0xb3, 0xb5, 0xc4, 0xbf, + 0x9d, 0xad, 0x25, 0xf6, 0xb2, 0x22, 0x08, 0xfa, 0xe4, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x7d, + 0xfe, 0xa7, 0xa7, 0xa8, 0x26, 0x00, 0x00, } diff --git a/api/types.proto b/api/types.proto index 88d8f87434..15c49c4a29 100644 --- a/api/types.proto +++ b/api/types.proto @@ -694,6 +694,13 @@ message RaftConfig { uint32 election_tick = 5; } +message EncryptionConfig { + // AutoLockManagers specifies whether or not managers TLS keys and raft data + // should be encrypted at rest in such a way that they must be unlocked + // before the manager node starts up again. + bool auto_lock_managers = 1; +} + // Placement specifies task distribution constraints. message Placement { // constraints specifies a set of requirements a node should meet for a task. diff --git a/ca/certificates.go b/ca/certificates.go index b4aa5fcf83..dc5dbc8f1c 100644 --- a/ca/certificates.go +++ b/ca/certificates.go @@ -28,6 +28,7 @@ import ( "github.com/pkg/errors" "golang.org/x/net/context" "google.golang.org/grpc" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" ) @@ -199,13 +200,62 @@ func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, kw KeyWrit return nil, err } - if err := kw.Write(signedCert, key, nil); err != nil { + var kekUpdate *KEKData + for i := 0; i < 5; i++ { + kekUpdate, err = rca.getKEKUpdate(ctx, X509Cert, tlsKeyPair, r) + if err == nil { + break + } + } + if err != nil { + return nil, err + } + + if err := kw.Write(signedCert, key, kekUpdate); err != nil { return nil, err } return &tlsKeyPair, nil } +func (rca *RootCA) getKEKUpdate(ctx context.Context, cert *x509.Certificate, keypair tls.Certificate, r remotes.Remotes) (*KEKData, error) { + var managerRole bool + for _, ou := range cert.Subject.OrganizationalUnit { + if ou == ManagerRole { + managerRole = true + break + } + } + + if managerRole { + mtlsCreds := credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rca.Pool, Certificates: []tls.Certificate{keypair}}) + conn, peer, err := getGRPCConnection(mtlsCreds, r) + if err != nil { + return nil, err + } + defer conn.Close() + + client := api.NewCAClient(conn) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + response, err := client.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{}) + if err != nil { + if grpc.Code(err) == codes.Unimplemented { // if the server does not support keks, return as if no encryption key was specified + return &KEKData{}, nil + } + + r.Observe(peer, -remotes.DefaultObservationWeight) + return nil, err + } + r.Observe(peer, remotes.DefaultObservationWeight) + return &KEKData{KEK: response.UnlockKey, Version: response.Version.Index}, nil + } + + // If this is a worker, set to never encrypt. We always want to set to the lock key to nil, + // in case this was a manager that was demoted to a worker. + return &KEKData{}, nil +} + // PrepareCSR creates a CFSSL Sign Request based on the given raw CSR and // overrides the Subject and Hosts with the given extra args. func PrepareCSR(csrBytes []byte, cn, ou, org string) cfsigner.SignRequest { @@ -390,24 +440,31 @@ func GetLocalRootCA(paths CertPaths) (RootCA, error) { return NewRootCA(cert, key, DefaultNodeCertExpiration) } -// GetRemoteCA returns the remote endpoint's CA certificate -func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) { - // This TLS Config is intentionally using InsecureSkipVerify. Either we're - // doing TOFU, in which case we don't validate the remote CA, or we're using - // a user supplied hash to check the integrity of the CA certificate. - insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) +func getGRPCConnection(creds credentials.TransportCredentials, r remotes.Remotes) (*grpc.ClientConn, api.Peer, error) { + peer, err := r.Select() + if err != nil { + return nil, api.Peer{}, err + } + opts := []grpc.DialOption{ - grpc.WithTransportCredentials(insecureCreds), + grpc.WithTransportCredentials(creds), grpc.WithTimeout(5 * time.Second), grpc.WithBackoffMaxDelay(5 * time.Second), } - peer, err := r.Select() + conn, err := grpc.Dial(peer.Addr, opts...) if err != nil { - return RootCA{}, err + return nil, api.Peer{}, err } + return conn, peer, nil +} - conn, err := grpc.Dial(peer.Addr, opts...) +// GetRemoteCA returns the remote endpoint's CA certificate +func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) { + // This TLS Config is intentionally using InsecureSkipVerify. We use the + // digest instead to check the integrity of the CA certificate. + insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) + conn, peer, err := getGRPCConnection(insecureCreds, r) if err != nil { return RootCA{}, err } @@ -499,18 +556,7 @@ func GetRemoteSignedCertificate(ctx context.Context, csr []byte, token string, r creds = credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rootCAPool}) } - peer, err := r.Select() - if err != nil { - return nil, err - } - - opts := []grpc.DialOption{ - grpc.WithTransportCredentials(creds), - grpc.WithTimeout(5 * time.Second), - grpc.WithBackoffMaxDelay(5 * time.Second), - } - - conn, err := grpc.Dial(peer.Addr, opts...) + conn, peer, err := getGRPCConnection(creds, r) if err != nil { return nil, err } diff --git a/ca/certificates_test.go b/ca/certificates_test.go index 9ce6fe3cd0..7a2e5a0ba9 100644 --- a/ca/certificates_test.go +++ b/ca/certificates_test.go @@ -296,10 +296,50 @@ func TestRequestAndSaveNewCertificates(t *testing.T) { assert.False(t, perms.OtherWrite()) assert.NotEmpty(t, <-info) - // the key should be unencrypted + // there was no encryption config in the remote, so the key should be unencrypted unencryptedKeyReader := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil) _, _, err = unencryptedKeyReader.Read() require.NoError(t, err) + + // the worker token is also unencrypted + cert, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.WorkerToken, tc.Remotes, nil, info) + assert.NoError(t, err) + assert.NotNil(t, cert) + assert.NotEmpty(t, <-info) + _, _, err = unencryptedKeyReader.Read() + require.NoError(t, err) + + // If there is a different kek in the remote store, when TLS certs are renewed the new key will + // be encrypted with that kek + assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error { + cluster := store.GetCluster(tx, tc.Organization) + cluster.Spec.EncryptionConfig.AutoLockManagers = true + cluster.UnlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: []byte("kek!"), + }} + return store.UpdateCluster(tx, cluster) + })) + assert.NoError(t, os.RemoveAll(tc.Paths.Node.Cert)) + assert.NoError(t, os.RemoveAll(tc.Paths.Node.Key)) + + _, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.ManagerToken, tc.Remotes, nil, info) + assert.NoError(t, err) + assert.NotEmpty(t, <-info) + + // key can no longer be read without a kek + _, _, err = unencryptedKeyReader.Read() + require.Error(t, err) + + _, _, err = ca.NewKeyReadWriter(tc.Paths.Node, []byte("kek!"), nil).Read() + require.NoError(t, err) + + // if it's a worker though, the key is always unencrypted, even though the manager key is encrypted + _, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter, tc.WorkerToken, tc.Remotes, nil, info) + assert.NoError(t, err) + assert.NotEmpty(t, <-info) + _, _, err = unencryptedKeyReader.Read() + require.NoError(t, err) } func TestIssueAndSaveNewCertificates(t *testing.T) { diff --git a/ca/server.go b/ca/server.go index 5b27e73880..938f597058 100644 --- a/ca/server.go +++ b/ca/server.go @@ -69,6 +69,33 @@ func (s *Server) SetReconciliationRetryInterval(reconciliationRetryInterval time s.reconciliationRetryInterval = reconciliationRetryInterval } +// GetUnlockKey is responsible for returning the current unlock key used for encrypting TLS private keys and +// other at rest data. Access to this RPC call should only be allowed via mutual TLS from managers. +func (s *Server) GetUnlockKey(ctx context.Context, request *api.GetUnlockKeyRequest) (*api.GetUnlockKeyResponse, error) { + // This directly queries the store, rather than storing the unlock key and version on + // the `Server` object and updating it `updateCluster` is called, because we need this + // API to return the latest version of the key. Otherwise, there might be a slight delay + // between when the cluster gets updated, and when this function returns the latest key. + // This delay is currently unacceptable because this RPC call is the only way, after a + // cluster update, to get the actual value of the unlock key, and we don't want to return + // a cached value. + resp := api.GetUnlockKeyResponse{} + s.store.View(func(tx store.ReadTx) { + cluster := store.GetCluster(tx, s.securityConfig.ClientTLSCreds.Organization()) + resp.Version = cluster.Meta.Version + if cluster.Spec.EncryptionConfig.AutoLockManagers { + for _, encryptionKey := range cluster.UnlockKeys { + if encryptionKey.Subsystem == ManagerRole { + resp.UnlockKey = encryptionKey.Key + return + } + } + } + }) + + return &resp, nil +} + // NodeCertificateStatus returns the current issuance status of an issuance request identified by the nodeID func (s *Server) NodeCertificateStatus(ctx context.Context, request *api.NodeCertificateStatusRequest) (*api.NodeCertificateStatusResponse, error) { if request.NodeID == "" { diff --git a/ca/server_test.go b/ca/server_test.go index 9a7ebd4dc8..3465f7372c 100644 --- a/ca/server_test.go +++ b/ca/server_test.go @@ -1,6 +1,8 @@ package ca_test import ( + "bytes" + "fmt" "testing" "time" @@ -9,6 +11,7 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/ca/testutils" + raftutils "github.com/docker/swarmkit/manager/state/raft/testutils" "github.com/docker/swarmkit/manager/state/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -350,3 +353,51 @@ func TestNewNodeCertificateBadToken(t *testing.T) { _, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest) assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster") } + +func TestGetUnlockKey(t *testing.T) { + t.Parallel() + + tc := testutils.NewTestCA(t) + defer tc.Stop() + + var cluster *api.Cluster + tc.MemoryStore.View(func(tx store.ReadTx) { + clusters, err := store.FindClusters(tx, store.ByName(store.DefaultClusterName)) + require.NoError(t, err) + cluster = clusters[0] + }) + + resp, err := tc.CAClients[0].GetUnlockKey(context.Background(), &api.GetUnlockKeyRequest{}) + require.NoError(t, err) + require.Nil(t, resp.UnlockKey) + require.Equal(t, cluster.Meta.Version, resp.Version) + + // Update the unlock key + require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error { + cluster = store.GetCluster(tx, cluster.ID) + cluster.Spec.EncryptionConfig.AutoLockManagers = true + cluster.UnlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: []byte("secret"), + }} + return store.UpdateCluster(tx, cluster) + })) + + tc.MemoryStore.View(func(tx store.ReadTx) { + cluster = store.GetCluster(tx, cluster.ID) + }) + + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + resp, err = tc.CAClients[0].GetUnlockKey(context.Background(), &api.GetUnlockKeyRequest{}) + if err != nil { + return fmt.Errorf("get unlock key: %v", err) + } + if !bytes.Equal(resp.UnlockKey, []byte("secret")) { + return fmt.Errorf("secret hasn't rotated yet") + } + if cluster.Meta.Version.Index > resp.Version.Index { + return fmt.Errorf("hasn't updated to the right version yet") + } + return nil + }, 250*time.Millisecond)) +} diff --git a/cmd/swarmctl/cluster/cmd.go b/cmd/swarmctl/cluster/cmd.go index 5635796fe8..8905418dac 100644 --- a/cmd/swarmctl/cluster/cmd.go +++ b/cmd/swarmctl/cluster/cmd.go @@ -15,5 +15,6 @@ func init() { inspectCmd, listCmd, updateCmd, + unlockKeyCmd, ) } diff --git a/cmd/swarmctl/cluster/unlockkey.go b/cmd/swarmctl/cluster/unlockkey.go new file mode 100644 index 0000000000..808dd56ad9 --- /dev/null +++ b/cmd/swarmctl/cluster/unlockkey.go @@ -0,0 +1,50 @@ +package cluster + +import ( + "errors" + "fmt" + + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/cmd/swarmctl/common" + "github.com/docker/swarmkit/manager/encryption" + "github.com/spf13/cobra" +) + +// get the unlock key + +func displayUnlockKey(cmd *cobra.Command) error { + conn, err := common.DialConn(cmd) + if err != nil { + return err + } + defer conn.Close() + + resp, err := api.NewCAClient(conn).GetUnlockKey(common.Context(cmd), &api.GetUnlockKeyRequest{}) + if err != nil { + return err + } + + if len(resp.UnlockKey) == 0 { + fmt.Printf("Managers not auto-locked") + } + fmt.Printf("Managers auto-locked. Unlock key: %s\n", encryption.HumanReadableKey(resp.UnlockKey)) + return nil +} + +var ( + unlockKeyCmd = &cobra.Command{ + Use: "unlock-key ", + Short: "Get the unlock key for a cluster", + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return errors.New("cluster name missing") + } + + if len(args) > 1 { + return errors.New("unlock-key command takes exactly 1 argument") + } + + return displayUnlockKey(cmd) + }, + } +) diff --git a/cmd/swarmctl/cluster/update.go b/cmd/swarmctl/cluster/update.go index 9e23f03c6f..cb5d81921d 100644 --- a/cmd/swarmctl/cluster/update.go +++ b/cmd/swarmctl/cluster/update.go @@ -3,6 +3,7 @@ package cluster import ( "errors" "fmt" + "strings" "time" "github.com/docker/swarmkit/api" @@ -39,7 +40,7 @@ var ( flags := cmd.Flags() spec := &cluster.Spec - var rotation api.JoinTokenRotation + var rotation api.KeyRotation if flags.Changed("certexpiry") { cePeriod, err := flags.GetDuration("certexpiry") @@ -71,16 +72,28 @@ var ( if err != nil { return err } + rotateJoinToken = strings.ToLower(rotateJoinToken) switch rotateJoinToken { case "worker": - rotation.RotateWorkerToken = true + rotation.WorkerJoinToken = true case "manager": - rotation.RotateManagerToken = true + rotation.ManagerJoinToken = true default: - return errors.New("--rotate-join-token flag must be followed by worker or manager") + return errors.New("--rotate-join-token flag must be followed by 'worker' or 'manager'") } } + if flags.Changed("autolock") { + spec.EncryptionConfig.AutoLockManagers, err = flags.GetBool("autolock") + if err != nil { + return err + } + } + rotateUnlockKey, err := flags.GetBool("rotate-unlock-key") + if err != nil { + return err + } + rotation.ManagerUnlockKey = rotateUnlockKey driver, err := common.ParseLogDriverFlags(flags) if err != nil { @@ -98,6 +111,10 @@ var ( return err } fmt.Println(r.Cluster.ID) + + if rotation.ManagerUnlockKey { + return displayUnlockKey(cmd) + } return nil }, } @@ -112,4 +129,6 @@ func init() { updateCmd.Flags().String("log-driver", "", "Set default log driver for cluster") updateCmd.Flags().StringSlice("log-opt", nil, "Set options for default log driver") updateCmd.Flags().String("rotate-join-token", "", "Rotate join token for worker or manager") + updateCmd.Flags().Bool("rotate-unlock-key", false, "Rotate manager unlock key") + updateCmd.Flags().Bool("autolock", false, "Enable or disable manager autolocking (requiring an unlock key to start a stopped manager)") } diff --git a/cmd/swarmd/main.go b/cmd/swarmd/main.go index 035b78b043..7e09db0cea 100644 --- a/cmd/swarmd/main.go +++ b/cmd/swarmd/main.go @@ -14,6 +14,7 @@ import ( "github.com/docker/swarmkit/agent/exec/container" "github.com/docker/swarmkit/cli" "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/node" "github.com/docker/swarmkit/version" "github.com/spf13/cobra" @@ -117,6 +118,23 @@ var ( return err } + autolockManagers, err := cmd.Flags().GetBool("autolock") + if err != nil { + return err + } + + var unlockKey []byte + if cmd.Flags().Changed("unlock-key") { + unlockKeyString, err := cmd.Flags().GetString("unlock-key") + if err != nil { + return err + } + unlockKey, err = encryption.ParseHumanReadableKey(unlockKeyString) + if err != nil { + return err + } + } + // Create a cancellable context for our GRPC call ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -149,6 +167,8 @@ var ( Executor: executor, HeartbeatTick: hb, ElectionTick: election, + AutoLockManagers: autolockManagers, + UnlockKey: unlockKey, }) if err != nil { return err @@ -195,4 +215,6 @@ func init() { mainCmd.Flags().Uint32("heartbeat-tick", 1, "Defines the heartbeat interval (in seconds) for raft member health-check") mainCmd.Flags().Uint32("election-tick", 3, "Defines the amount of ticks (in seconds) needed without a Leader to trigger a new election") mainCmd.Flags().Var(&externalCAOpt, "external-ca", "Specifications of one or more certificate signing endpoints") + mainCmd.Flags().Bool("autolock", false, "Require an unlock key in order to start a manager once it's been stopped") + mainCmd.Flags().String("unlock-key", "", "Unlock this manager using this key") } diff --git a/manager/controlapi/cluster.go b/manager/controlapi/cluster.go index fd3cb3df99..f2015886e4 100644 --- a/manager/controlapi/cluster.go +++ b/manager/controlapi/cluster.go @@ -6,6 +6,7 @@ import ( "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "golang.org/x/net/context" @@ -107,12 +108,38 @@ func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRe expireBlacklistedCerts(cluster) - if request.Rotation.RotateWorkerToken { + if request.Rotation.WorkerJoinToken { cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA) } - if request.Rotation.RotateManagerToken { + if request.Rotation.ManagerJoinToken { cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA) } + + var unlockKeys []*api.EncryptionKey + var managerKey *api.EncryptionKey + for _, eKey := range cluster.UnlockKeys { + if eKey.Subsystem == ca.ManagerRole { + if !cluster.Spec.EncryptionConfig.AutoLockManagers { + continue + } + managerKey = eKey + } + unlockKeys = append(unlockKeys, eKey) + } + + switch { + case !cluster.Spec.EncryptionConfig.AutoLockManagers: + break + case managerKey == nil: + unlockKeys = append(unlockKeys, &api.EncryptionKey{ + Subsystem: ca.ManagerRole, + Key: encryption.GenerateSecretKey(), + }) + case request.Rotation.ManagerUnlockKey: + managerKey.Key = encryption.GenerateSecretKey() + } + cluster.UnlockKeys = unlockKeys + return store.UpdateCluster(tx, cluster) }) if err != nil { diff --git a/manager/controlapi/cluster_test.go b/manager/controlapi/cluster_test.go index fe3c8f3906..85c433815e 100644 --- a/manager/controlapi/cluster_test.go +++ b/manager/controlapi/cluster_test.go @@ -1,6 +1,7 @@ package controlapi import ( + "fmt" "testing" "time" @@ -9,6 +10,7 @@ import ( "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/net/context" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -25,11 +27,11 @@ func createClusterSpec(name string) *api.ClusterSpec { } } -func createCluster(t *testing.T, ts *testServer, id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster { +func createClusterObj(id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster { spec := createClusterSpec(name) spec.AcceptancePolicy = policy - cluster := &api.Cluster{ + return &api.Cluster{ ID: id, Spec: *spec, RootCA: api.RootCA{ @@ -42,6 +44,10 @@ func createCluster(t *testing.T, ts *testServer, id, name string, policy api.Acc }, }, } +} + +func createCluster(t *testing.T, ts *testServer, id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster { + cluster := createClusterObj(id, name, policy, rootCA) assert.NoError(t, ts.Store.Update(func(tx store.Tx) error { return store.CreateCluster(tx, cluster) })) @@ -244,8 +250,8 @@ func TestUpdateClusterRotateToken(t *testing.T) { ClusterID: cluster.ID, Spec: &cluster.Spec, ClusterVersion: &cluster.Meta.Version, - Rotation: api.JoinTokenRotation{ - RotateWorkerToken: true, + Rotation: api.KeyRotation{ + WorkerJoinToken: true, }, }) assert.NoError(t, err) @@ -266,8 +272,8 @@ func TestUpdateClusterRotateToken(t *testing.T) { ClusterID: cluster.ID, Spec: &cluster.Spec, ClusterVersion: &r.Clusters[0].Meta.Version, - Rotation: api.JoinTokenRotation{ - RotateManagerToken: true, + Rotation: api.KeyRotation{ + ManagerJoinToken: true, }, }) assert.NoError(t, err) @@ -288,9 +294,9 @@ func TestUpdateClusterRotateToken(t *testing.T) { ClusterID: cluster.ID, Spec: &cluster.Spec, ClusterVersion: &r.Clusters[0].Meta.Version, - Rotation: api.JoinTokenRotation{ - RotateWorkerToken: true, - RotateManagerToken: true, + Rotation: api.KeyRotation{ + WorkerJoinToken: true, + ManagerJoinToken: true, }, }) assert.NoError(t, err) @@ -306,6 +312,126 @@ func TestUpdateClusterRotateToken(t *testing.T) { assert.NotEqual(t, managerToken, r.Clusters[0].RootCA.JoinTokens.Manager) } +func TestUpdateClusterRotateUnlockKey(t *testing.T) { + ts := newTestServer(t) + defer ts.Stop() + // create a cluster with extra encryption keys, to make sure they exist + cluster := createClusterObj("id", "name", api.AcceptancePolicy{}, ts.Server.rootCA) + expected := make(map[string]*api.EncryptionKey) + for i := 1; i <= 2; i++ { + value := fmt.Sprintf("fake%d", i) + expected[value] = &api.EncryptionKey{Subsystem: value, Key: []byte(value)} + cluster.UnlockKeys = append(cluster.UnlockKeys, expected[value]) + } + require.NoError(t, ts.Store.Update(func(tx store.Tx) error { + return store.CreateCluster(tx, cluster) + })) + + // we have to get the key from the memory store, since the cluster returned by the API is redacted + getManagerKey := func() (managerKey *api.EncryptionKey) { + ts.Store.View(func(tx store.ReadTx) { + viewCluster := store.GetCluster(tx, cluster.ID) + // no matter whether there's a manager key or not, the other keys should not have been affected + foundKeys := make(map[string]*api.EncryptionKey) + for _, eKey := range viewCluster.UnlockKeys { + foundKeys[eKey.Subsystem] = eKey + } + for v, key := range expected { + foundKey, ok := foundKeys[v] + require.True(t, ok) + require.Equal(t, key, foundKey) + } + managerKey = foundKeys[ca.ManagerRole] + }) + return + } + + validateListResult := func(expectedLocked bool) api.Version { + r, err := ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{ + Filters: &api.ListClustersRequest_Filters{ + NamePrefixes: []string{"name"}, + }, + }) + + require.NoError(t, err) + require.Len(t, r.Clusters, 1) + require.Equal(t, expectedLocked, r.Clusters[0].Spec.EncryptionConfig.AutoLockManagers) + require.Nil(t, r.Clusters[0].UnlockKeys) // redacted + + return r.Clusters[0].Meta.Version + } + + // we start off with manager autolocking turned off + version := validateListResult(false) + require.Nil(t, getManagerKey()) + + // Rotate unlock key without turning auto-lock on - key should still be nil + _, err := ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: &cluster.Spec, + ClusterVersion: &version, + Rotation: api.KeyRotation{ + ManagerUnlockKey: true, + }, + }) + require.NoError(t, err) + version = validateListResult(false) + require.Nil(t, getManagerKey()) + + // Enable auto-lock only, no rotation boolean + spec := cluster.Spec.Copy() + spec.EncryptionConfig.AutoLockManagers = true + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: spec, + ClusterVersion: &version, + }) + require.NoError(t, err) + version = validateListResult(true) + managerKey := getManagerKey() + require.NotNil(t, managerKey) + + // Rotate the manager key + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: spec, + ClusterVersion: &version, + Rotation: api.KeyRotation{ + ManagerUnlockKey: true, + }, + }) + require.NoError(t, err) + version = validateListResult(true) + newManagerKey := getManagerKey() + require.NotNil(t, managerKey) + require.NotEqual(t, managerKey, newManagerKey) + managerKey = newManagerKey + + // Just update the cluster without modifying unlock keys + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: spec, + ClusterVersion: &version, + }) + require.NoError(t, err) + version = validateListResult(true) + newManagerKey = getManagerKey() + require.Equal(t, managerKey, newManagerKey) + + // Disable auto lock + _, err = ts.Client.UpdateCluster(context.Background(), &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + Spec: &cluster.Spec, // set back to original spec + ClusterVersion: &version, + Rotation: api.KeyRotation{ + ManagerUnlockKey: true, // this will be ignored because we disable the auto-lock + }, + }) + require.NoError(t, err) + validateListResult(false) + require.Nil(t, getManagerKey()) +} + func TestListClusters(t *testing.T) { ts := newTestServer(t) defer ts.Stop() diff --git a/manager/encryption/encryption.go b/manager/encryption/encryption.go index 7ce834c1d7..313e0e2bbd 100644 --- a/manager/encryption/encryption.go +++ b/manager/encryption/encryption.go @@ -115,7 +115,7 @@ func GenerateSecretKey() []byte { // HumanReadableKey displays a secret key in a human readable way func HumanReadableKey(key []byte) string { // base64-encode the key - return humanReadablePrefix + base64.StdEncoding.EncodeToString(key) + return humanReadablePrefix + base64.RawStdEncoding.EncodeToString(key) } // ParseHumanReadableKey returns a key as bytes from recognized serializations of @@ -124,7 +124,7 @@ func ParseHumanReadableKey(key string) ([]byte, error) { if !strings.HasPrefix(key, humanReadablePrefix) { return nil, fmt.Errorf("invalid key string") } - keyBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(key, humanReadablePrefix)) + keyBytes, err := base64.RawStdEncoding.DecodeString(strings.TrimPrefix(key, humanReadablePrefix)) if err != nil { return nil, fmt.Errorf("invalid key string") } diff --git a/manager/encryption/encryption_test.go b/manager/encryption/encryption_test.go index ef98a606b9..14f2d9a728 100644 --- a/manager/encryption/encryption_test.go +++ b/manager/encryption/encryption_test.go @@ -62,7 +62,7 @@ func TestHumanReadable(t *testing.T) { require.Error(t, err) // With the right prefix, we can't parse if the key isn't base64 encoded - _, err = ParseHumanReadableKey(humanReadablePrefix + "aaaaa/") + _, err = ParseHumanReadableKey(humanReadablePrefix + "aaa*aa/") require.Error(t, err) // Extra padding also fails diff --git a/manager/manager.go b/manager/manager.go index 0c10565423..39c734360c 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -86,6 +86,17 @@ type Config struct { // HeartbeatTick defines the amount of ticks between each // heartbeat sent to other members for health-check purposes HeartbeatTick uint32 + + // AutoLockManagers determines whether or not managers require an unlock key + // when starting from a stopped state. This configuration parameter is only + // applicable when bootstrapping a new cluster for the first time. + AutoLockManagers bool + + // UnlockKey is the key to unlock a node - used for decrypting manager TLS keys + // as well as the raft data encryption key (DEK). It is applicable when + // bootstrapping a cluster for the first time (it's a cluster-wide setting), + // and also when loading up any raft data on disk (as a KEK for the raft DEK). + UnlockKey []byte } // Manager is the cluster manager for Swarm. @@ -320,6 +331,7 @@ func (m *Manager) Run(parent context.Context) error { forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil } localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, m.raftNode, forwardAsOwnRequest) localProxyLogsAPI := api.NewRaftProxyLogsServer(m.logbroker, m.raftNode, forwardAsOwnRequest) + localCAAPI := api.NewRaftProxyCAServer(m.caserver, m.raftNode, forwardAsOwnRequest) // Everything registered on m.server should be an authenticated // wrapper, or a proxy wrapping an authenticated wrapper! @@ -337,6 +349,7 @@ func (m *Manager) Run(parent context.Context) error { api.RegisterControlServer(m.localserver, localProxyControlAPI) api.RegisterLogsServer(m.localserver, localProxyLogsAPI) api.RegisterHealthServer(m.localserver, localHealthServer) + api.RegisterCAServer(m.localserver, localCAAPI) healthServer.SetServingStatus("Raft", api.HealthCheckResponse_NOT_SERVING) localHealthServer.SetServingStatus("ControlAPI", api.HealthCheckResponse_NOT_SERVING) @@ -625,12 +638,26 @@ func (m *Manager) becomeLeader(ctx context.Context) { initialCAConfig := ca.DefaultCAConfig() initialCAConfig.ExternalCAs = m.config.ExternalCAs + var unlockKeys []*api.EncryptionKey + if m.config.AutoLockManagers && m.config.UnlockKey != nil { + unlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: m.config.UnlockKey, + }} + } + s.Update(func(tx store.Tx) error { // Add a default cluster object to the // store. Don't check the error because // we expect this to fail unless this // is a brand new cluster. - store.CreateCluster(tx, defaultClusterObject(clusterID, initialCAConfig, raftCfg, rootCA)) + store.CreateCluster(tx, defaultClusterObject( + clusterID, + initialCAConfig, + raftCfg, + api.EncryptionConfig{AutoLockManagers: m.config.AutoLockManagers}, + unlockKeys, + rootCA)) // Add Node entry for ourself, if one // doesn't exist already. store.CreateNode(tx, managerNode(nodeID)) @@ -759,7 +786,14 @@ func (m *Manager) becomeFollower() { } // defaultClusterObject creates a default cluster. -func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCfg api.RaftConfig, rootCA *ca.RootCA) *api.Cluster { +func defaultClusterObject( + clusterID string, + initialCAConfig api.CAConfig, + raftCfg api.RaftConfig, + encryptionConfig api.EncryptionConfig, + initialUnlockKeys []*api.EncryptionKey, + rootCA *ca.RootCA, +) *api.Cluster { return &api.Cluster{ ID: clusterID, Spec: api.ClusterSpec{ @@ -772,8 +806,9 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf Dispatcher: api.DispatcherConfig{ HeartbeatPeriod: ptypes.DurationProto(dispatcher.DefaultHeartBeatPeriod), }, - Raft: raftCfg, - CAConfig: initialCAConfig, + Raft: raftCfg, + CAConfig: initialCAConfig, + EncryptionConfig: encryptionConfig, }, RootCA: api.RootCA{ CAKey: rootCA.Key, @@ -784,6 +819,7 @@ func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCf Manager: ca.GenerateJoinToken(rootCA), }, }, + UnlockKeys: initialUnlockKeys, } } diff --git a/manager/manager_test.go b/manager/manager_test.go index 990672a451..de724b95d3 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -18,6 +18,7 @@ import ( "github.com/docker/swarmkit/manager/dispatcher" "github.com/docker/swarmkit/manager/state/store" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestManager(t *testing.T) { @@ -45,10 +46,12 @@ func TestManager(t *testing.T) { assert.NoError(t, err) m, err := New(&Config{ - RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, - ControlAPI: temp.Name(), - StateDir: stateDir, - SecurityConfig: managerSecurityConfig, + RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, + ControlAPI: temp.Name(), + StateDir: stateDir, + SecurityConfig: managerSecurityConfig, + UnlockKey: []byte("kek"), + AutoLockManagers: true, }) assert.NoError(t, err) assert.NotNil(t, m) @@ -131,6 +134,21 @@ func TestManager(t *testing.T) { assert.NoError(t, controlConn.Close()) }() + // check that the kek is added to the config + var cluster api.Cluster + m.raftNode.MemoryStore().View(func(tx store.ReadTx) { + clusters, err := store.FindClusters(tx, store.All) + require.NoError(t, err) + require.Len(t, clusters, 1) + cluster = *clusters[0] + }) + require.NotNil(t, cluster) + require.Len(t, cluster.UnlockKeys, 1) + require.Equal(t, &api.EncryptionKey{ + Subsystem: ca.ManagerRole, + Key: []byte("kek"), + }, cluster.UnlockKeys[0]) + // Test removal of the agent node agentID := agentSecurityConfig.ClientTLSCreds.NodeID() assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error { diff --git a/node/node.go b/node/node.go index 606c41c342..1baa7216a6 100644 --- a/node/node.go +++ b/node/node.go @@ -664,13 +664,15 @@ func (n *Node) runManager(ctx context.Context, securityConfig *ca.SecurityConfig ListenAddr: n.config.ListenRemoteAPI, AdvertiseAddr: n.config.AdvertiseRemoteAPI, }, - ControlAPI: n.config.ListenControlAPI, - SecurityConfig: securityConfig, - ExternalCAs: n.config.ExternalCAs, - JoinRaft: remoteAddr.Addr, - StateDir: n.config.StateDir, - HeartbeatTick: n.config.HeartbeatTick, - ElectionTick: n.config.ElectionTick, + ControlAPI: n.config.ListenControlAPI, + SecurityConfig: securityConfig, + ExternalCAs: n.config.ExternalCAs, + JoinRaft: remoteAddr.Addr, + StateDir: n.config.StateDir, + HeartbeatTick: n.config.HeartbeatTick, + ElectionTick: n.config.ElectionTick, + AutoLockManagers: n.config.AutoLockManagers, + UnlockKey: n.unlockKey, }) if err != nil { return err From 31b94a7d6fb0a3d6e15243159d92b5d6bf01bebc Mon Sep 17 00:00:00 2001 From: cyli Date: Tue, 8 Nov 2016 16:04:21 -0800 Subject: [PATCH 3/3] Refactor all the raft/storage bits that actually write to the wal and snapshot into a storage package that knows how to do encryption and rotate encryption keys. Handle raft DEK encryption rotation in the raft node and in the manager. Ensure that when a node starts up, it uses a PEMKeyHeader interface that knows how to handle raft DEKs. Signed-off-by: cyli --- ca/testutils/cautils.go | 5 +- manager/deks.go | 269 ++++++++++++ manager/deks_test.go | 464 ++++++++++++++++++++ manager/manager.go | 96 +++- manager/manager_test.go | 222 +++++++++- manager/state/raft/raft.go | 107 +++-- manager/state/raft/raft_test.go | 8 + manager/state/raft/storage.go | 445 +++---------------- manager/state/raft/storage/snapwrap.go | 63 +++ manager/state/raft/storage/snapwrap_test.go | 57 +++ manager/state/raft/storage/storage.go | 391 +++++++++++++++++ manager/state/raft/storage/storage_test.go | 221 ++++++++++ manager/state/raft/storage/walwrap.go | 123 ++++++ manager/state/raft/storage/walwrap_test.go | 124 +++++- manager/state/raft/storage_test.go | 250 ++++++++++- manager/state/raft/testutils/testutils.go | 109 ++++- node/node.go | 9 +- node/node_test.go | 31 ++ 18 files changed, 2534 insertions(+), 460 deletions(-) create mode 100644 manager/deks.go create mode 100644 manager/deks_test.go create mode 100644 manager/state/raft/storage/storage.go create mode 100644 manager/state/raft/storage/storage_test.go diff --git a/ca/testutils/cautils.go b/ca/testutils/cautils.go index e7158a6598..d249825c70 100644 --- a/ca/testutils/cautils.go +++ b/ca/testutils/cautils.go @@ -95,7 +95,7 @@ var External bool // NewTestCA is a helper method that creates a TestCA and a bunch of default // connections and security configs. -func NewTestCA(t *testing.T) *TestCA { +func NewTestCA(t *testing.T, krwGenerators ...func(ca.CertPaths) *ca.KeyReadWriter) *TestCA { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) @@ -125,6 +125,9 @@ func NewTestCA(t *testing.T) *TestCA { } krw := ca.NewKeyReadWriter(paths.Node, nil, nil) + if len(krwGenerators) > 0 { + krw = krwGenerators[0](paths.Node) + } managerConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, organization, "", External) assert.NoError(t, err) diff --git a/manager/deks.go b/manager/deks.go new file mode 100644 index 0000000000..4813a67d53 --- /dev/null +++ b/manager/deks.go @@ -0,0 +1,269 @@ +package manager + +import ( + "crypto/subtle" + "encoding/base64" + "fmt" + + "github.com/docker/swarmkit/ca" + "github.com/docker/swarmkit/manager/encryption" + "github.com/docker/swarmkit/manager/state/raft" +) + +const ( + // the raft DEK (data encryption key) is stored in the TLS key as a header + // these are the header values + pemHeaderRaftDEK = "raft-dek" + pemHeaderRaftPendingDEK = "raft-dek-pending" + pemHeaderRaftDEKNeedsRotation = "raft-dek-needs-rotation" +) + +// RaftDEKData contains all the data stored in TLS pem headers +type RaftDEKData struct { + raft.EncryptionKeys + NeedsRotation bool +} + +// UnmarshalHeaders loads the state of the DEK manager given the current TLS headers +func (r RaftDEKData) UnmarshalHeaders(headers map[string]string, kekData ca.KEKData) (ca.PEMKeyHeaders, error) { + var ( + currentDEK, pendingDEK []byte + err error + ) + + if currentDEKStr, ok := headers[pemHeaderRaftDEK]; ok { + currentDEK, err = decodePEMHeaderValue(currentDEKStr, kekData.KEK) + if err != nil { + return nil, err + } + } + if pendingDEKStr, ok := headers[pemHeaderRaftPendingDEK]; ok { + pendingDEK, err = decodePEMHeaderValue(pendingDEKStr, kekData.KEK) + if err != nil { + return nil, err + } + } + + if pendingDEK != nil && currentDEK == nil { + return nil, fmt.Errorf("there is a pending DEK, but no current DEK") + } + + _, ok := headers[pemHeaderRaftDEKNeedsRotation] + return RaftDEKData{ + NeedsRotation: ok, + EncryptionKeys: raft.EncryptionKeys{ + CurrentDEK: currentDEK, + PendingDEK: pendingDEK, + }, + }, nil +} + +// MarshalHeaders returns new headers given the current KEK +func (r RaftDEKData) MarshalHeaders(kekData ca.KEKData) (map[string]string, error) { + headers := make(map[string]string) + for headerKey, contents := range map[string][]byte{ + pemHeaderRaftDEK: r.CurrentDEK, + pemHeaderRaftPendingDEK: r.PendingDEK, + } { + if contents != nil { + dekStr, err := encodePEMHeaderValue(contents, kekData.KEK) + if err != nil { + return nil, err + } + headers[headerKey] = dekStr + } + } + + if r.NeedsRotation { + headers[pemHeaderRaftDEKNeedsRotation] = "true" + } + + // return a function that updates the dek data on write success + return headers, nil +} + +// UpdateKEK optionally sets NeedRotation to true if we go from unlocked to locked +func (r RaftDEKData) UpdateKEK(oldKEK, candidateKEK ca.KEKData) ca.PEMKeyHeaders { + if _, unlockedToLocked, err := compareKEKs(oldKEK, candidateKEK); err == nil && unlockedToLocked { + return RaftDEKData{ + EncryptionKeys: r.EncryptionKeys, + NeedsRotation: true, + } + } + return r +} + +// Returns whether the old KEK should be replaced with the new KEK, whether we went from +// unlocked to locked, and whether there was an error (the versions are the same, but the +// keks are different) +func compareKEKs(oldKEK, candidateKEK ca.KEKData) (bool, bool, error) { + keksEqual := subtle.ConstantTimeCompare(oldKEK.KEK, candidateKEK.KEK) == 1 + switch { + case oldKEK.Version == candidateKEK.Version && !keksEqual: + return false, false, fmt.Errorf("candidate KEK has the same version as the current KEK, but a different KEK value") + case oldKEK.Version >= candidateKEK.Version || keksEqual: + return false, false, nil + default: + return true, oldKEK.KEK == nil, nil + } +} + +// RaftDEKManager manages the raft DEK keys using TLS headers +type RaftDEKManager struct { + kw ca.KeyWriter + rotationCh chan struct{} +} + +var errNoUpdateNeeded = fmt.Errorf("don't need to rotate or update") + +// this error is returned if the KeyReadWriter's PEMKeyHeaders object is no longer a RaftDEKData object - +// this can happen if the node is no longer a manager, for example +var errNotUsingRaftDEKData = fmt.Errorf("RaftDEKManager can no longer store and manage TLS key headers") + +// NewRaftDEKManager returns a RaftDEKManager that uses the current key writer +// and header manager +func NewRaftDEKManager(kw ca.KeyWriter) (*RaftDEKManager, error) { + // If there is no current DEK, generate one and write it to disk + err := kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + dekData, ok := h.(RaftDEKData) + // it wasn't a raft DEK manager before - just replace it + if !ok || dekData.CurrentDEK == nil { + return RaftDEKData{ + EncryptionKeys: raft.EncryptionKeys{ + CurrentDEK: encryption.GenerateSecretKey(), + }, + }, nil + } + return nil, errNoUpdateNeeded + }) + if err != nil && err != errNoUpdateNeeded { + return nil, err + } + return &RaftDEKManager{ + kw: kw, + rotationCh: make(chan struct{}, 1), + }, nil +} + +// NeedsRotation returns a boolean about whether we should do a rotation +func (r *RaftDEKManager) NeedsRotation() bool { + h, _ := r.kw.GetCurrentState() + data, ok := h.(RaftDEKData) + if !ok { + return false + } + return data.NeedsRotation || data.EncryptionKeys.PendingDEK != nil +} + +// GetKeys returns the current set of DEKs. If NeedsRotation is true, and there +// is no existing PendingDEK, it will try to create one. If there are any errors +// doing so, just return the original. +func (r *RaftDEKManager) GetKeys() raft.EncryptionKeys { + var newKeys, originalKeys raft.EncryptionKeys + err := r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + data, ok := h.(RaftDEKData) + if !ok { + return nil, errNotUsingRaftDEKData + } + originalKeys = data.EncryptionKeys + if !data.NeedsRotation || data.PendingDEK != nil { + return nil, errNoUpdateNeeded + } + newKeys = raft.EncryptionKeys{ + CurrentDEK: data.CurrentDEK, + PendingDEK: encryption.GenerateSecretKey(), + } + return RaftDEKData{EncryptionKeys: newKeys}, nil + }) + if err != nil { + return originalKeys + } + return newKeys +} + +// RotationNotify the channel used to notify subscribers as to whether there +// should be a rotation done +func (r *RaftDEKManager) RotationNotify() chan struct{} { + return r.rotationCh +} + +// UpdateKeys will set the updated encryption keys in the headers. This finishes +// a rotation, and is expected to set the CurrentDEK to the previous PendingDEK. +func (r *RaftDEKManager) UpdateKeys(newKeys raft.EncryptionKeys) error { + return r.kw.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) { + data, ok := h.(RaftDEKData) + if !ok { + return nil, errNotUsingRaftDEKData + } + // If there is no current DEK, we are basically wiping out all DEKs (no header object) + if newKeys.CurrentDEK == nil { + return nil, nil + } + return RaftDEKData{ + EncryptionKeys: newKeys, + NeedsRotation: data.NeedsRotation, + }, nil + }) +} + +// MaybeUpdateKEK does a KEK rotation if one is required. Returns whether +// the kek was updated, whether it went from unlocked to locked, and any errors. +func (r *RaftDEKManager) MaybeUpdateKEK(candidateKEK ca.KEKData) (bool, bool, error) { + var updated, unlockedToLocked bool + err := r.kw.ViewAndRotateKEK(func(currentKEK ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) { + var err error + updated, unlockedToLocked, err = compareKEKs(currentKEK, candidateKEK) + if err == nil && !updated { // if we don't need to rotate the KEK, don't bother updating + err = errNoUpdateNeeded + } + if err != nil { + return ca.KEKData{}, nil, err + } + + data, ok := h.(RaftDEKData) + if !ok { + return ca.KEKData{}, nil, errNotUsingRaftDEKData + } + + if unlockedToLocked { + data.NeedsRotation = true + } + return candidateKEK, data, nil + }) + if err == errNoUpdateNeeded { + err = nil + } + + if err == nil && unlockedToLocked { + r.rotationCh <- struct{}{} + } + return updated, unlockedToLocked, err +} + +func decodePEMHeaderValue(headerValue string, kek []byte) ([]byte, error) { + var decrypter encryption.Decrypter = encryption.NoopCrypter + if kek != nil { + _, decrypter = encryption.Defaults(kek) + } + valueBytes, err := base64.StdEncoding.DecodeString(headerValue) + if err != nil { + return nil, err + } + result, err := encryption.Decrypt(valueBytes, decrypter) + if err != nil { + return nil, ca.ErrInvalidKEK{Wrapped: err} + } + return result, nil +} + +func encodePEMHeaderValue(headerValue []byte, kek []byte) (string, error) { + var encrypter encryption.Encrypter = encryption.NoopCrypter + if kek != nil { + encrypter, _ = encryption.Defaults(kek) + } + encrypted, err := encryption.Encrypt(headerValue, encrypter) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(encrypted), nil +} diff --git a/manager/deks_test.go b/manager/deks_test.go new file mode 100644 index 0000000000..55ed40debb --- /dev/null +++ b/manager/deks_test.go @@ -0,0 +1,464 @@ +package manager + +import ( + "encoding/base64" + "encoding/pem" + "io/ioutil" + "os" + "testing" + + "github.com/docker/swarmkit/ca" + cautils "github.com/docker/swarmkit/ca/testutils" + "github.com/docker/swarmkit/manager/state/raft" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +// Tests updating a kek on a raftDEK object. +func TestRaftDEKUpdateKEK(t *testing.T) { + startData := RaftDEKData{ + EncryptionKeys: raft.EncryptionKeys{CurrentDEK: []byte("first dek")}, + } + startKEK := ca.KEKData{} + + // because UpdateKEK returns a PEMKeyHeaders interface, we need to cast to check + // values + updateDEKAndCast := func(dekdata RaftDEKData, oldKEK ca.KEKData, newKEK ca.KEKData) RaftDEKData { + result := dekdata.UpdateKEK(oldKEK, newKEK) + raftDekObj, ok := result.(RaftDEKData) + require.True(t, ok) + return raftDekObj + } + + // nothing changes if we are updating a kek and they're both nil + result := updateDEKAndCast(startData, startKEK, ca.KEKData{Version: 2}) + require.Equal(t, result, startData) + + // when moving from unlocked to locked, a "needs rotation" header is generated but no + // pending header is generated + updatedKEK := ca.KEKData{KEK: []byte("something"), Version: 1} + result = updateDEKAndCast(startData, startKEK, updatedKEK) + require.NotEqual(t, startData, result) + require.True(t, result.NeedsRotation) + require.Equal(t, startData.CurrentDEK, result.CurrentDEK) + require.Nil(t, result.PendingDEK) + + // this is whether or not pending exists + startData.PendingDEK = []byte("pending") + result = updateDEKAndCast(startData, startKEK, updatedKEK) + require.NotEqual(t, startData, result) + require.True(t, result.NeedsRotation) + require.Equal(t, startData.CurrentDEK, result.CurrentDEK) + require.Equal(t, startData.PendingDEK, result.PendingDEK) + + // if we are going from locked to unlocked, nothing happens + result = updateDEKAndCast(startData, updatedKEK, startKEK) + require.Equal(t, startData, result) + require.False(t, result.NeedsRotation) + + // if we are going to locked to another locked, nothing happens + result = updateDEKAndCast(startData, updatedKEK, ca.KEKData{KEK: []byte("other"), Version: 4}) + require.Equal(t, startData, result) + require.False(t, result.NeedsRotation) +} + +func TestRaftDEKMarshalUnmarshal(t *testing.T) { + startData := RaftDEKData{ + EncryptionKeys: raft.EncryptionKeys{CurrentDEK: []byte("first dek")}, + } + kek := ca.KEKData{} + + headers, err := startData.MarshalHeaders(kek) + require.NoError(t, err) + require.Len(t, headers, 1) + + // can't unmarshal with the wrong kek + _, err = RaftDEKData{}.UnmarshalHeaders(headers, ca.KEKData{KEK: []byte("something")}) + require.Error(t, err) + + // we can unmarshal what was marshalled with the right kek + toData, err := RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.NoError(t, err) + require.Equal(t, startData, toData) + + // try the other headers as well + startData.PendingDEK = []byte("Hello") + headers, err = startData.MarshalHeaders(kek) + require.NoError(t, err) + require.Len(t, headers, 2) + + // we can unmarshal what was marshalled + toData, err = RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.NoError(t, err) + require.Equal(t, startData, toData) + + // try the other headers as well + startData.NeedsRotation = true + startData.PendingDEK = nil + headers, err = startData.MarshalHeaders(kek) + require.NoError(t, err) + require.Len(t, headers, 2) + + // we can unmarshal what was marshalled + toData, err = RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.NoError(t, err) + require.Equal(t, startData, toData) + + // If there is a pending header, but no current header, set will fail + headers = map[string]string{ + pemHeaderRaftPendingDEK: headers[pemHeaderRaftDEK], + } + _, err = RaftDEKData{}.UnmarshalHeaders(headers, kek) + require.Error(t, err) + require.Contains(t, err.Error(), "pending DEK, but no current DEK") +} + +// NewRaftDEKManager creates a key if one doesn't exist +func TestNewRaftDEKManager(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-new-dek-manager-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + krw := ca.NewKeyReadWriter(paths.Node, nil, nil) + require.NoError(t, krw.Write(cert, key, nil)) + + keyBytes, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.NotContains(t, string(keyBytes), pemHeaderRaftDEK) // headers are not written + + dekManager, err := NewRaftDEKManager(krw) // this should create a new DEK and write it to the file + require.NoError(t, err) + + keyBytes, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Contains(t, string(keyBytes), pemHeaderRaftDEK) // header is written now + + keys := dekManager.GetKeys() + require.NotNil(t, keys.CurrentDEK) + require.Nil(t, keys.PendingDEK) + require.False(t, dekManager.NeedsRotation()) + + // If one exists, nothing is updated + dekManager, err = NewRaftDEKManager(krw) // this should create a new DEK and write it to the file + require.NoError(t, err) + + keyBytes2, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2) + + require.Equal(t, keys, dekManager.GetKeys()) + require.False(t, dekManager.NeedsRotation()) +} + +// NeedsRotate returns true if there is a PendingDEK or a NeedsRotation flag +func TestRaftDEKManagerNeedsRotateGetKeys(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-maybe-get-data-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + + // if there is no PendingDEK, and no NeedsRotation flag: NeedsRotation=false + keys := raft.EncryptionKeys{CurrentDEK: []byte("hello")} + dekManager, err := NewRaftDEKManager( + ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{EncryptionKeys: keys})) + require.NoError(t, err) + + require.False(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + + // if there is a PendingDEK, and no NeedsRotation flag: NeedsRotation=true + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello"), PendingDEK: []byte("another")} + dekManager, err = NewRaftDEKManager( + ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{EncryptionKeys: keys})) + require.NoError(t, err) + + require.True(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + + // if there is a PendingDEK, and a NeedsRotation flag: NeedsRotation=true + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello"), PendingDEK: []byte("another")} + dekManager, err = NewRaftDEKManager( + ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + })) + require.NoError(t, err) + + require.True(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + + // if there no PendingDEK, and a NeedsRotation flag: NeedsRotation=true and + // GetKeys attempts to create a pending key and write it to disk. However, writing + // will error (because there is no key on disk atm), and then the original keys will + // be returned. + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello")} + krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + }) + dekManager, err = NewRaftDEKManager(krw) + require.NoError(t, err) + + require.True(t, dekManager.NeedsRotation()) + require.Equal(t, keys, dekManager.GetKeys()) + h, _ := krw.GetCurrentState() + dekData, ok := h.(RaftDEKData) + require.True(t, ok) + require.True(t, dekData.NeedsRotation) + + // if there no PendingDEK, and a NeedsRotation flag: NeedsRotation=true and + // GetKeys attempts to create a pending key and write it to disk. If successful, + // it retuns the new keys + keys = raft.EncryptionKeys{CurrentDEK: []byte("hello")} + krw = ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + }) + dekManager, err = NewRaftDEKManager(krw) + + require.NoError(t, err) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + require.NoError(t, krw.Write(cert, key, nil)) + + require.True(t, dekManager.NeedsRotation()) + updatedKeys := dekManager.GetKeys() + require.Equal(t, keys.CurrentDEK, updatedKeys.CurrentDEK) + require.NotNil(t, updatedKeys.PendingDEK) + require.True(t, dekManager.NeedsRotation()) + + h, _ = krw.GetCurrentState() + dekData, ok = h.(RaftDEKData) + require.True(t, ok) + require.False(t, dekData.NeedsRotation) +} + +func TestRaftDEKManagerUpdateKeys(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-update-keys-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + keys := raft.EncryptionKeys{ + CurrentDEK: []byte("key1"), + PendingDEK: []byte("key2"), + } + krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{ + EncryptionKeys: keys, + NeedsRotation: true, + }) + require.NoError(t, krw.Write(cert, key, nil)) + + dekManager, err := NewRaftDEKManager(krw) + require.NoError(t, err) + + newKeys := raft.EncryptionKeys{ + CurrentDEK: []byte("new current"), + } + require.NoError(t, dekManager.UpdateKeys(newKeys)) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + + h, _ := krw.GetCurrentState() + dekData, ok := h.(RaftDEKData) + require.True(t, ok) + require.True(t, dekData.NeedsRotation) + + // UpdateKeys so there is no CurrentDEK: all the headers should be wiped out + require.NoError(t, dekManager.UpdateKeys(raft.EncryptionKeys{})) + require.Equal(t, raft.EncryptionKeys{}, dekManager.GetKeys()) + require.False(t, dekManager.NeedsRotation()) + + h, _ = krw.GetCurrentState() + require.Nil(t, h) + + keyBytes, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + keyBlock, _ := pem.Decode(keyBytes) + require.NotNil(t, keyBlock) + + // the only header remaining should be the kek version + require.Len(t, keyBlock.Headers, 1) + require.Contains(t, keyBlock.Headers, "kek-version") +} + +func TestRaftDEKManagerMaybeUpdateKEK(t *testing.T) { + tempDir, err := ioutil.TempDir("", "manager-maybe-update-kek-") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + paths := ca.NewConfigPaths(tempDir) + cert, key, err := cautils.CreateRootCertAndKey("cn") + require.NoError(t, err) + + keys := raft.EncryptionKeys{CurrentDEK: []byte("current dek")} + + // trying to update a KEK will error if the version is the same but the kek is different + krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{EncryptionKeys: keys}) + require.NoError(t, krw.Write(cert, key, nil)) + dekManager, err := NewRaftDEKManager(krw) + require.NoError(t, err) + + keyBytes, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + + _, _, err = dekManager.MaybeUpdateKEK(ca.KEKData{KEK: []byte("locked now")}) + require.Error(t, err) + require.False(t, dekManager.NeedsRotation()) + + keyBytes2, err := ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2) + + // trying to update a KEK from unlocked to lock will set NeedsRotation to true, as well as encrypt the TLS key + updated, unlockedToLocked, err := dekManager.MaybeUpdateKEK(ca.KEKData{KEK: []byte("locked now"), Version: 1}) + require.NoError(t, err) + require.True(t, updated) + require.True(t, unlockedToLocked) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + h, _ := krw.GetCurrentState() + dekData, ok := h.(RaftDEKData) + require.True(t, ok) + require.Equal(t, keys, dekData.EncryptionKeys) + require.True(t, dekData.NeedsRotation) + require.NotNil(t, <-dekManager.RotationNotify()) // we are notified of a new pending key + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.NotEqual(t, keyBytes, keyBytes2) + keyBytes = keyBytes2 + + readKRW := ca.NewKeyReadWriter(paths.Node, []byte("locked now"), RaftDEKData{}) + _, _, err = readKRW.Read() + require.NoError(t, err) + + // trying to update a KEK of a lower version will not update anything, but will not error + updated, unlockedToLocked, err = dekManager.MaybeUpdateKEK(ca.KEKData{}) + require.NoError(t, err) + require.False(t, unlockedToLocked) + require.False(t, updated) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + h, _ = krw.GetCurrentState() + dekData, ok = h.(RaftDEKData) + require.True(t, ok) + require.Equal(t, keys, dekData.EncryptionKeys) + require.True(t, dekData.NeedsRotation) + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2, string(keyBytes), string(keyBytes2)) + + // updating a kek to a higher version, but with the same kek, will also neither update anything nor error + updated, unlockedToLocked, err = dekManager.MaybeUpdateKEK(ca.KEKData{KEK: []byte("locked now"), Version: 100}) + require.NoError(t, err) + require.False(t, unlockedToLocked) + require.False(t, updated) + // don't run GetKeys, because NeedsRotation is true and it'd just generate a new one + h, _ = krw.GetCurrentState() + dekData, ok = h.(RaftDEKData) + require.True(t, ok) + require.Equal(t, keys, dekData.EncryptionKeys) + require.True(t, dekData.NeedsRotation) + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.Equal(t, keyBytes, keyBytes2) + + // going from locked to unlock does not result in the NeedsRotation flag, but does result in + // the key being decrypted + krw = ca.NewKeyReadWriter(paths.Node, []byte("kek"), RaftDEKData{EncryptionKeys: keys}) + require.NoError(t, krw.Write(cert, key, nil)) + dekManager, err = NewRaftDEKManager(krw) + require.NoError(t, err) + + keyBytes, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + + updated, unlockedToLocked, err = dekManager.MaybeUpdateKEK(ca.KEKData{Version: 2}) + require.NoError(t, err) + require.False(t, unlockedToLocked) + require.True(t, updated) + require.Equal(t, keys, dekManager.GetKeys()) + require.False(t, dekManager.NeedsRotation()) + + keyBytes2, err = ioutil.ReadFile(paths.Node.Key) + require.NoError(t, err) + require.NotEqual(t, keyBytes, keyBytes2) + + readKRW = ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{}) + _, _, err = readKRW.Read() + require.NoError(t, err) +} + +// The TLS KEK and the KEK for the headers should be in sync, and so failing +// to decrypt the TLS key should be mean we won't be able to decrypt the headers. +// However, the TLS Key encryption uses AES-256-CBC (golang as of 1.7.x does not seem +// to support GCM, so no cipher modes with digests) so sometimes decrypting with +// the wrong passphrase will not result in an error. This means we will ultimately +// have to rely on the header encryption mechanism, which does include a digest, to +// determine if the KEK is valid. +func TestDecryptTLSKeyFalsePositive(t *testing.T) { + badKey := []byte(` +-----BEGIN EC PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-256-CBC,e7927e79e748233776c03c2eb7275f09 +kek-version: 392 +raft-dek: CAESMBrzZ0gNVPe3FRs42743q8RtkUBrK1ICQpHWX2vdQ8iqSKt1WoKdFDFD2r28LYAVLxoYQguwHbijMx9k+BALUNBAI3s199S5tvnr + +JfGenNvzm++AvsOh+UmcBY+JgI6lnfzaCB68agmlmEZYLYi5tqtAU7gif6VIJpCW ++Pj23Fzkw8sKKOOBeapSC5lp+Cjx9OsCci/R9xrdx+uxnnzKJNxOB/qzqcQfZDMh +id2LxdliFcPEk/Yj5gNGpT0UMFJ4G52enbOwOru46f0= +-----END EC PRIVATE KEY----- +`) + + // not actually a real swarm cert - generated a cert corresponding to the key that expires in 20 years + matchingCert := []byte(` +-----BEGIN CERTIFICATE----- +MIIB9jCCAZygAwIBAgIRAIdzF3Z9VT2OXbRvEw5cR68wCgYIKoZIzj0EAwIwYDEi +MCAGA1UEChMZbWRwMXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dh +cm0tbWFuYWdlcjEiMCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTAg +GA8wMDAxMDEwMTAwMDAwMFoXDTM2MTEwODA2MjMwMlowYDEiMCAGA1UEChMZbWRw +MXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dhcm0tbWFuYWdlcjEi +MCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTBZMBMGByqGSM49AgEG +CCqGSM49AwEHA0IABGOivD25E/zcupRFQdKOKbPHS9Mx7JlUhlWnl0iR0K5VhVIU +XjUHt98GuX6gDjs4yUzEKSGxYPsSYlnG9zQqbQSjNTAzMA4GA1UdDwEB/wQEAwIF +oDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMAoGCCqGSM49BAMC +A0gAMEUCIQDWtjg1ITGznQILipaEe70G/NgZAOtFfuPXTVkUl3el+wIgSVOVKB/Q +O0T3aXuZGYNyh//KqAoA3erCmh6HauMz84Y= +-----END CERTIFICATE----- + `) + + var wrongKEK []byte // empty passphrase doesn't decrypt without errors + falsePositiveKEK, err := base64.RawStdEncoding.DecodeString("bIQgLAAMoGCrHdjMLVhEVqnYTAM7ZNF2xWMiwtw7AiQ") + require.NoError(t, err) + realKEK, err := base64.RawStdEncoding.DecodeString("fDg9YejLnMjU+FpulWR62oJLzVpkD2j7VQuP5xiK9QA") + require.NoError(t, err) + + tempdir, err := ioutil.TempDir("", "KeyReadWriter-false-positive-decryption") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + path := ca.NewConfigPaths(tempdir) + require.NoError(t, ioutil.WriteFile(path.Node.Key, badKey, 0600)) + require.NoError(t, ioutil.WriteFile(path.Node.Cert, matchingCert, 0644)) + + krw := ca.NewKeyReadWriter(path.Node, wrongKEK, RaftDEKData{}) + _, _, err = krw.Read() + require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err)) + + krw = ca.NewKeyReadWriter(path.Node, falsePositiveKEK, RaftDEKData{}) + _, _, err = krw.Read() + require.Error(t, err) + require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err)) + + krw = ca.NewKeyReadWriter(path.Node, realKEK, RaftDEKData{}) + _, _, err = krw.Read() + require.NoError(t, err) +} diff --git a/manager/manager.go b/manager/manager.go index 39c734360c..975ca57fc5 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -29,9 +29,11 @@ import ( "github.com/docker/swarmkit/manager/orchestrator/taskreaper" "github.com/docker/swarmkit/manager/resourceapi" "github.com/docker/swarmkit/manager/scheduler" + "github.com/docker/swarmkit/manager/state" "github.com/docker/swarmkit/manager/state/raft" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/protobuf/ptypes" + "github.com/docker/swarmkit/remotes" "github.com/docker/swarmkit/xnet" "github.com/pkg/errors" "golang.org/x/net/context" @@ -119,6 +121,7 @@ type Manager struct { server *grpc.Server localserver *grpc.Server raftNode *raft.Node + dekRotator *RaftDEKManager cancelFunc context.CancelFunc @@ -228,6 +231,11 @@ func New(config *Config) (*Manager, error) { raftCfg.HeartbeatTick = int(config.HeartbeatTick) } + dekRotator, err := NewRaftDEKManager(config.SecurityConfig.KeyWriter()) + if err != nil { + return nil, err + } + newNodeOpts := raft.NodeOptions{ ID: config.SecurityConfig.ClientTLSCreds.NodeID(), Addr: advertiseAddr, @@ -236,6 +244,7 @@ func New(config *Config) (*Manager, error) { StateDir: raftStateDir, ForceNewCluster: config.ForceNewCluster, TLSCredentials: config.SecurityConfig.ClientTLSCreds, + KeyRotator: dekRotator, } raftNode := raft.NewNode(newNodeOpts) @@ -252,6 +261,7 @@ func New(config *Config) (*Manager, error) { localserver: grpc.NewServer(opts...), raftNode: raftNode, started: make(chan struct{}), + dekRotator: dekRotator, } return m, nil @@ -375,8 +385,12 @@ func (m *Manager) Run(parent context.Context) error { close(m.started) + watchDone := make(chan struct{}) + watchCtx, watchCtxCancel := context.WithCancel(parent) go func() { err := m.raftNode.Run(ctx) + watchCtxCancel() + <-watchDone if err != nil { log.G(ctx).Error(err) m.Stop(ctx) @@ -393,6 +407,10 @@ func (m *Manager) Run(parent context.Context) error { } raftConfig := c.Spec.Raft + if err := m.watchForKEKChanges(watchCtx, watchDone); err != nil { + return err + } + if int(raftConfig.ElectionTick) != m.raftNode.Config.ElectionTick { log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.raftNode.Config.ElectionTick, raftConfig.ElectionTick) } @@ -488,6 +506,78 @@ func (m *Manager) Stop(ctx context.Context) { // mutex is released and Run can return now } +func (m *Manager) updateKEK(ctx context.Context, cluster *api.Cluster) error { + securityConfig := m.config.SecurityConfig + nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID() + logger := log.G(ctx).WithFields(logrus.Fields{ + "node.id": nodeID, + "node.role": ca.ManagerRole, + }) + + // we are our own peer from which we get certs - try to connect over the local socket + r := remotes.NewRemotes(api.Peer{Addr: m.Addr(), NodeID: nodeID}) + + kekData := ca.KEKData{Version: cluster.Meta.Version.Index} + for _, encryptionKey := range cluster.UnlockKeys { + if encryptionKey.Subsystem == ca.ManagerRole { + kekData.KEK = encryptionKey.Key + break + } + } + updated, unlockedToLocked, err := m.dekRotator.MaybeUpdateKEK(kekData) + if err != nil { + logger.WithError(err).Errorf("failed to re-encrypt TLS key with a new KEK") + return err + } + if updated { + logger.Debug("successfully rotated KEK") + } + if unlockedToLocked { + // a best effort attempt to update the TLS certificate - if it fails, it'll be updated the next time it renews; + // don't wait because it might take a bit + go func() { + if err := ca.RenewTLSConfigNow(ctx, securityConfig, r); err != nil { + logger.WithError(err).Errorf("failed to download new TLS certificate after locking the cluster") + } + }() + } + return nil +} + +func (m *Manager) watchForKEKChanges(ctx context.Context, watchDone chan struct{}) error { + defer close(watchDone) + clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization() + clusterWatch, clusterWatchCancel, err := store.ViewAndWatch(m.raftNode.MemoryStore(), + func(tx store.ReadTx) error { + cluster := store.GetCluster(tx, clusterID) + if cluster == nil { + return fmt.Errorf("unable to get current cluster") + } + return m.updateKEK(ctx, cluster) + }, + state.EventUpdateCluster{ + Cluster: &api.Cluster{ID: clusterID}, + Checks: []state.ClusterCheckFunc{state.ClusterCheckID}, + }, + ) + if err != nil { + return err + } + go func() { + for { + select { + case event := <-clusterWatch: + clusterEvent := event.(state.EventUpdateCluster) + m.updateKEK(ctx, clusterEvent.Cluster) + case <-ctx.Done(): + clusterWatchCancel() + return + } + } + }() + return nil +} + // rotateRootCAKEK will attempt to rotate the key-encryption-key for root CA key-material in raft. // If there is no passphrase set in ENV, it returns. // If there is plain-text root key-material, and a passphrase set, it encrypts it. @@ -639,7 +729,7 @@ func (m *Manager) becomeLeader(ctx context.Context) { initialCAConfig.ExternalCAs = m.config.ExternalCAs var unlockKeys []*api.EncryptionKey - if m.config.AutoLockManagers && m.config.UnlockKey != nil { + if m.config.AutoLockManagers { unlockKeys = []*api.EncryptionKey{{ Subsystem: ca.ManagerRole, Key: m.config.UnlockKey, @@ -792,8 +882,8 @@ func defaultClusterObject( raftCfg api.RaftConfig, encryptionConfig api.EncryptionConfig, initialUnlockKeys []*api.EncryptionKey, - rootCA *ca.RootCA, -) *api.Cluster { + rootCA *ca.RootCA) *api.Cluster { + return &api.Cluster{ ID: clusterID, Spec: api.ClusterSpec{ diff --git a/manager/manager_test.go b/manager/manager_test.go index de724b95d3..e34cf716d6 100644 --- a/manager/manager_test.go +++ b/manager/manager_test.go @@ -1,9 +1,14 @@ package manager import ( + "bytes" "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" "io/ioutil" "os" + "path/filepath" "testing" "time" @@ -16,6 +21,9 @@ import ( "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/ca/testutils" "github.com/docker/swarmkit/manager/dispatcher" + "github.com/docker/swarmkit/manager/encryption" + "github.com/docker/swarmkit/manager/state/raft/storage" + raftutils "github.com/docker/swarmkit/manager/state/raft/testutils" "github.com/docker/swarmkit/manager/state/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,7 +43,9 @@ func TestManager(t *testing.T) { assert.NoError(t, err) defer os.RemoveAll(stateDir) - tc := testutils.NewTestCA(t) + tc := testutils.NewTestCA(t, func(p ca.CertPaths) *ca.KeyReadWriter { + return ca.NewKeyReadWriter(p, []byte("kek"), nil) + }) defer tc.Stop() agentSecurityConfig, err := tc.NewNodeConfig(ca.WorkerRole) @@ -50,8 +60,8 @@ func TestManager(t *testing.T) { ControlAPI: temp.Name(), StateDir: stateDir, SecurityConfig: managerSecurityConfig, - UnlockKey: []byte("kek"), AutoLockManagers: true, + UnlockKey: []byte("kek"), }) assert.NoError(t, err) assert.NotNil(t, m) @@ -182,3 +192,211 @@ func TestManager(t *testing.T) { // error. <-done } + +// Tests locking and unlocking the manager and key rotations +func TestManagerLockUnlock(t *testing.T) { + ctx := context.Background() + + temp, err := ioutil.TempFile("", "test-manager-lock") + require.NoError(t, err) + require.NoError(t, temp.Close()) + require.NoError(t, os.Remove(temp.Name())) + + defer os.RemoveAll(temp.Name()) + + stateDir, err := ioutil.TempDir("", "test-raft") + require.NoError(t, err) + defer os.RemoveAll(stateDir) + + tc := testutils.NewTestCA(t) + defer tc.Stop() + + managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole) + require.NoError(t, err) + + _, _, err = managerSecurityConfig.KeyReader().Read() + require.NoError(t, err) + + m, err := New(&Config{ + RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, + ControlAPI: temp.Name(), + StateDir: stateDir, + SecurityConfig: managerSecurityConfig, + // start off without any encryption + }) + require.NoError(t, err) + require.NotNil(t, m) + + done := make(chan error) + defer close(done) + go func() { + done <- m.Run(ctx) + }() + + opts := []grpc.DialOption{ + grpc.WithTimeout(10 * time.Second), + grpc.WithTransportCredentials(managerSecurityConfig.ClientTLSCreds), + } + + conn, err := grpc.Dial(m.Addr(), opts...) + require.NoError(t, err) + defer func() { + require.NoError(t, conn.Close()) + }() + + // check that there is no kek currently - we are using the API because this + // lets us wait until the manager is up and listening, as well + var cluster *api.Cluster + client := api.NewControlClient(conn) + + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + resp, err := client.ListClusters(ctx, &api.ListClustersRequest{}) + if err != nil { + return err + } + if len(resp.Clusters) == 0 { + return fmt.Errorf("no clusters yet") + } + cluster = resp.Clusters[0] + return nil + }, 1*time.Second)) + + require.Nil(t, cluster.UnlockKeys) + + // tls key is unencrypted, but there is a DEK + key, err := ioutil.ReadFile(tc.Paths.Node.Key) + require.NoError(t, err) + keyBlock, _ := pem.Decode(key) + require.NotNil(t, keyBlock) + require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) + require.Len(t, keyBlock.Headers, 2) + currentDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) + require.NoError(t, err) + require.NotEmpty(t, currentDEK) + + // update the lock key - this may fail due to update out of sequence errors, so try again + for { + getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) + require.NoError(t, err) + cluster = getResp.Cluster + + spec := cluster.Spec.Copy() + spec.EncryptionConfig.AutoLockManagers = true + updateResp, err := client.UpdateCluster(ctx, &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + ClusterVersion: &cluster.Meta.Version, + Spec: spec, + }) + if grpc.ErrorDesc(err) == "update out of sequence" { + continue + } + // if there is any other type of error, this should fail + if err == nil { + cluster = updateResp.Cluster + } + break + } + require.NoError(t, err) + + caConn := api.NewCAClient(conn) + unlockKeyResp, err := caConn.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{}) + require.NoError(t, err) + + // this should update the TLS key, rotate the DEK, and finish snapshotting + var updatedKey []byte + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + updatedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) + require.NoError(t, err) // this should never error due to atomic writes + + if bytes.Equal(key, updatedKey) { + return fmt.Errorf("TLS key should have been re-encrypted at least") + } + + keyBlock, _ = pem.Decode(updatedKey) + require.NotNil(t, keyBlock) // this should never error due to atomic writes + + if !x509.IsEncryptedPEMBlock(keyBlock) { + return fmt.Errorf("Key not encrypted") + } + + // we don't check that the TLS key has been rotated, because that may take + // a little bit, and is best effort only + + currentDEKString, ok := keyBlock.Headers[pemHeaderRaftDEK] + require.True(t, ok) // there should never NOT be a current header + nowCurrentDEK, err := decodePEMHeaderValue(currentDEKString, unlockKeyResp.UnlockKey) + require.NoError(t, err) // it should always be encrypted + if bytes.Equal(currentDEK, nowCurrentDEK) { + return fmt.Errorf("snapshot has not been finished yet") + } + + currentDEK = nowCurrentDEK + return nil + }, 1*time.Second)) + + _, ok := keyBlock.Headers[pemHeaderRaftPendingDEK] + require.False(t, ok) // once the snapshot is do + + _, ok = keyBlock.Headers[pemHeaderRaftDEKNeedsRotation] + require.False(t, ok) + + // verify that the snapshot is readable with the new DEK + encrypter, decrypter := encryption.Defaults(currentDEK) + // we can't use the raftLogger, because the WALs are still locked while the raft node is up. And once we remove + // the manager, they'll be deleted. + snapshot, err := storage.NewSnapFactory(encrypter, decrypter).New(filepath.Join(stateDir, "raft", "snap-v3-encrypted")).Load() + require.NoError(t, err) + require.NotNil(t, snapshot) + + // update the lock key to nil + for i := 0; i < 3; i++ { + getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) + require.NoError(t, err) + cluster = getResp.Cluster + + spec := cluster.Spec.Copy() + spec.EncryptionConfig.AutoLockManagers = false + _, err = client.UpdateCluster(ctx, &api.UpdateClusterRequest{ + ClusterID: cluster.ID, + ClusterVersion: &cluster.Meta.Version, + Spec: spec, + }) + if grpc.ErrorDesc(err) == "update out of sequence" { + continue + } + require.NoError(t, err) + } + + // this should update the TLS key + var unlockedKey []byte + require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { + unlockedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) + if err != nil { + return err + } + + if bytes.Equal(unlockedKey, updatedKey) { + return fmt.Errorf("TLS key should have been rotated") + } + + return nil + }, 1*time.Second)) + + // the new key should not be encrypted, and the DEK should also be unencrypted + // but not rotated + keyBlock, _ = pem.Decode(unlockedKey) + require.NotNil(t, keyBlock) + require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) + + unencryptedDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) + require.NoError(t, err) + require.NotNil(t, unencryptedDEK) + require.Equal(t, currentDEK, unencryptedDEK) + + m.Stop(ctx) + + // After stopping we should MAY receive an error from ListenAndServe if + // all this happened before WaitForLeader completed, so don't check the + // error. + <-done +} diff --git a/manager/state/raft/raft.go b/manager/state/raft/raft.go index 42cf6d40e2..c3ff5a8432 100644 --- a/manager/state/raft/raft.go +++ b/manager/state/raft/raft.go @@ -20,14 +20,13 @@ import ( "github.com/coreos/etcd/pkg/idutil" "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" - "github.com/coreos/etcd/snap" - "github.com/coreos/etcd/wal" "github.com/docker/go-events" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/raftselector" "github.com/docker/swarmkit/manager/state/raft/membership" + "github.com/docker/swarmkit/manager/state/raft/storage" "github.com/docker/swarmkit/manager/state/store" "github.com/docker/swarmkit/watch" "github.com/gogo/protobuf/proto" @@ -75,6 +74,21 @@ const ( IsFollower ) +// EncryptionKeys are the current and, if necessary, pending DEKs with which to +// encrypt raft data +type EncryptionKeys struct { + CurrentDEK []byte + PendingDEK []byte +} + +// EncryptionKeyRotator is an interface to find out if any keys need rotating. +type EncryptionKeyRotator interface { + GetKeys() EncryptionKeys + UpdateKeys(EncryptionKeys) error + NeedsRotation() bool + RotationNotify() chan struct{} +} + // Node represents the Raft Node useful // configuration. type Node struct { @@ -87,8 +101,6 @@ type Node struct { opts NodeOptions reqIDGen *idutil.Generator wait *wait - wal *wal.WAL - snapshotter *snap.Snapshotter campaignWhenAble bool signalledLeadership uint32 isMember uint32 @@ -122,6 +134,9 @@ type Node struct { stopped chan struct{} lastSendToMember map[uint64]chan struct{} + raftLogger *storage.EncryptedRaftLogger + keyRotator EncryptionKeyRotator + rotationQueued bool } // NodeOptions provides node-level options. @@ -150,6 +165,8 @@ type NodeOptions struct { // nodes. Leave this as 0 to get the default value. SendTimeout time.Duration TLSCredentials credentials.TransportCredentials + + KeyRotator EncryptionKeyRotator } func init() { @@ -188,6 +205,7 @@ func NewNode(opts NodeOptions) *Node { stopped: make(chan struct{}), leadershipBroadcast: watch.NewQueue(), lastSendToMember: make(map[uint64]chan struct{}), + keyRotator: opts.KeyRotator, } n.memoryStore = store.NewMemoryStore(n) @@ -238,7 +256,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) { }() loadAndStartErr := n.loadAndStart(ctx, n.opts.ForceNewCluster) - if loadAndStartErr != nil && loadAndStartErr != errNoWAL { + if loadAndStartErr != nil && loadAndStartErr != storage.ErrNoWAL { return loadAndStartErr } @@ -252,7 +270,7 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) { n.appliedIndex = snapshot.Metadata.Index n.snapshotIndex = snapshot.Metadata.Index - if loadAndStartErr == errNoWAL { + if loadAndStartErr == storage.ErrNoWAL { if n.opts.JoinAddr != "" { c, err := n.ConnectToMember(n.opts.JoinAddr, 10*time.Second) if err != nil { @@ -274,22 +292,20 @@ func (n *Node) JoinAndStart(ctx context.Context) (err error) { n.Config.ID = resp.RaftID - if _, err := n.createWAL(n.opts.ID); err != nil { + if _, err := n.newRaftLogs(n.opts.ID); err != nil { return err } n.raftNode = raft.StartNode(n.Config, []raft.Peer{}) if err := n.registerNodes(resp.Members); err != nil { - if walErr := n.wal.Close(); err != nil { - log.G(ctx).WithError(walErr).Error("raft: error closing WAL") - } + n.raftLogger.Close(ctx) return err } } else { // First member in the cluster, self-assign ID n.Config.ID = uint64(rand.Int63()) + 1 - peer, err := n.createWAL(n.opts.ID) + peer, err := n.newRaftLogs(n.opts.ID) if err != nil { return err } @@ -367,9 +383,13 @@ func (n *Node) Run(ctx context.Context) error { if nodeRemoved { // Move WAL and snapshot out of the way, since // they are no longer usable. - if err := n.moveWALAndSnap(); err != nil { + if err := n.raftLogger.Clear(ctx); err != nil { log.G(ctx).WithError(err).Error("failed to move wal after node removal") } + // clear out the DEKs + if err := n.keyRotator.UpdateKeys(EncryptionKeys{}); err != nil { + log.G(ctx).WithError(err).Error("could not remove DEKs") + } } n.done() }() @@ -382,16 +402,10 @@ func (n *Node) Run(ctx context.Context) error { n.raftNode.Tick() n.cluster.Tick() case rd := <-n.raftNode.Ready(): - raftConfig := DefaultRaftConfig() - n.memoryStore.View(func(readTx store.ReadTx) { - clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) - if err == nil && len(clusters) == 1 { - raftConfig = clusters[0].Spec.Raft - } - }) + raftConfig := n.getCurrentRaftConfig() // Save entries to storage - if err := n.saveToStorage(&raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil { + if err := n.saveToStorage(ctx, &raftConfig, rd.HardState, rd.Entries, rd.Snapshot); err != nil { log.G(ctx).WithError(err).Error("failed to save entries to storage") } @@ -459,8 +473,8 @@ func (n *Node) Run(ctx context.Context) error { // Trigger a snapshot every once in awhile if n.snapshotInProgress == nil && - raftConfig.SnapshotInterval > 0 && - n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval { + (n.keyRotator.NeedsRotation() || raftConfig.SnapshotInterval > 0 && + n.appliedIndex-n.snapshotIndex >= raftConfig.SnapshotInterval) { n.doSnapshot(ctx, raftConfig) } @@ -496,6 +510,24 @@ func (n *Node) Run(ctx context.Context) error { n.snapshotIndex = snapshotIndex } n.snapshotInProgress = nil + if n.rotationQueued { + // there was a key rotation that took place before while the snapshot + // was in progress - we have to take another snapshot and encrypt with the new key + n.doSnapshot(ctx, n.getCurrentRaftConfig()) + } + case <-n.keyRotator.RotationNotify(): + // There are 2 separate checks: rotationQueued, and keyRotator.NeedsRotation(). + // We set rotationQueued so that when we are notified of a rotation, we try to + // do a snapshot as soon as possible. However, if there is an error while doing + // the snapshot, we don't want to hammer the node attempting to do snapshots over + // and over. So if doing a snapshot fails, wait until the next entry comes in to + // try again. + switch { + case n.snapshotInProgress != nil: + n.rotationQueued = true + case n.keyRotator.NeedsRotation(): + n.doSnapshot(ctx, n.getCurrentRaftConfig()) + } case <-n.removeRaftCh: nodeRemoved = true // If the node was removed from other members, @@ -508,6 +540,17 @@ func (n *Node) Run(ctx context.Context) error { } } +func (n *Node) getCurrentRaftConfig() api.RaftConfig { + raftConfig := DefaultRaftConfig() + n.memoryStore.View(func(readTx store.ReadTx) { + clusters, err := store.FindClusters(readTx, store.ByName(store.DefaultClusterName)) + if err == nil && len(clusters) == 1 { + raftConfig = clusters[0].Spec.Raft + } + }) + return raftConfig +} + // Done returns channel which is closed when raft node is fully stopped. func (n *Node) Done() <-chan struct{} { return n.doneCh @@ -524,9 +567,7 @@ func (n *Node) stop(ctx context.Context) { n.raftNode.Stop() n.ticker.Stop() - if err := n.wal.Close(); err != nil { - log.G(ctx).WithError(err).Error("raft: failed to close WAL") - } + n.raftLogger.Close(ctx) atomic.StoreUint32(&n.isMember, 0) // TODO(stevvooe): Handle ctx.Done() } @@ -1123,17 +1164,27 @@ func (n *Node) canSubmitProposal() bool { } // Saves a log entry to our Store -func (n *Node) saveToStorage(raftConfig *api.RaftConfig, hardState raftpb.HardState, entries []raftpb.Entry, snapshot raftpb.Snapshot) (err error) { +func (n *Node) saveToStorage( + ctx context.Context, + raftConfig *api.RaftConfig, + hardState raftpb.HardState, + entries []raftpb.Entry, + snapshot raftpb.Snapshot, +) (err error) { + if !raft.IsEmptySnap(snapshot) { - if err := n.saveSnapshot(snapshot, raftConfig.KeepOldSnapshots); err != nil { + if err := n.raftLogger.SaveSnapshot(snapshot); err != nil { return ErrApplySnapshot } + if err := n.raftLogger.GC(snapshot.Metadata.Index, snapshot.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil { + log.G(ctx).WithError(err).Error("unable to clean old snapshots and WALs") + } if err = n.raftStore.ApplySnapshot(snapshot); err != nil { return ErrApplySnapshot } } - if err := n.wal.Save(hardState, entries); err != nil { + if err := n.raftLogger.SaveEntries(hardState, entries); err != nil { // TODO(aaronl): These error types should really wrap more // detailed errors. return ErrApplySnapshot diff --git a/manager/state/raft/raft_test.go b/manager/state/raft/raft_test.go index 7ff390a6c0..c91336b2ba 100644 --- a/manager/state/raft/raft_test.go +++ b/manager/state/raft/raft_test.go @@ -6,6 +6,7 @@ import ( "log" "math/rand" "os" + "path/filepath" "reflect" "strconv" "testing" @@ -317,6 +318,13 @@ func TestRaftLeaderLeave(t *testing.T) { // Wait for election tick raftutils.WaitForCluster(t, clockSource, newCluster) + // Node1's state should be cleared + _, err = os.Stat(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted")) + require.True(t, os.IsNotExist(err)) + _, err = os.Stat(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted")) + require.True(t, os.IsNotExist(err)) + require.Equal(t, raft.EncryptionKeys{}, nodes[1].KeyRotator.GetKeys()) + // Leader should not be 1 assert.NotEqual(t, nodes[2].Leader(), nodes[1].Config.ID) assert.Equal(t, nodes[2].Leader(), nodes[3].Leader()) diff --git a/manager/state/raft/storage.go b/manager/state/raft/storage.go index c47da6ce68..8a7dd422c1 100644 --- a/manager/state/raft/storage.go +++ b/manager/state/raft/storage.go @@ -2,282 +2,72 @@ package raft import ( "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/coreos/etcd/pkg/fileutil" + "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" - "github.com/coreos/etcd/snap" - "github.com/coreos/etcd/wal" - "github.com/coreos/etcd/wal/walpb" "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/encryption" "github.com/docker/swarmkit/manager/state/raft/membership" + "github.com/docker/swarmkit/manager/state/raft/storage" "github.com/docker/swarmkit/manager/state/store" "github.com/pkg/errors" "golang.org/x/net/context" ) -var errNoWAL = errors.New("no WAL present") - -func (n *Node) legacyWALDir() string { - return filepath.Join(n.opts.StateDir, "wal") -} - -func (n *Node) walDir() string { - return filepath.Join(n.opts.StateDir, "wal-v3") -} - -func (n *Node) legacySnapDir() string { - return filepath.Join(n.opts.StateDir, "snap") -} - -func (n *Node) snapDir() string { - return filepath.Join(n.opts.StateDir, "snap-v3") -} - -func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error { - walDir := n.walDir() - snapDir := n.snapDir() - - if !fileutil.Exist(snapDir) { - // If snapshots created by the etcd-v2 code exist, hard link - // them at the new path. This prevents etc-v2 creating - // snapshots that are visible to us, but out of sync with our - // WALs, after a downgrade. - legacySnapDir := n.legacySnapDir() - if fileutil.Exist(legacySnapDir) { - if err := migrateSnapshots(legacySnapDir, snapDir); err != nil { - return err - } - } else if err := os.MkdirAll(snapDir, 0700); err != nil { - return errors.Wrap(err, "failed to create snapshot directory") - } - } - - // Create a snapshotter - n.snapshotter = snap.New(snapDir) - - if !wal.Exist(walDir) { - // If wals created by the etcd-v2 wal code exist, copy them to - // the new path to avoid adding backwards-incompatible entries - // to those files. - legacyWALDir := n.legacyWALDir() - if !wal.Exist(legacyWALDir) { - return errNoWAL - } - - if err := migrateWALs(legacyWALDir, walDir); err != nil { - return err - } - } - - // Load snapshot data - snapshot, err := n.snapshotter.Load() - if err != nil && err != snap.ErrNoSnapshot { - return err - } - - if snapshot != nil { - // Load the snapshot data into the store - if err := n.restoreFromSnapshot(snapshot.Data, forceNewCluster); err != nil { - return err - } - } - - // Read logs to fully catch up store - if err := n.readWAL(ctx, snapshot, forceNewCluster); err != nil { - return err - } - - return nil -} +func (n *Node) readFromDisk(ctx context.Context) (*raftpb.Snapshot, storage.WALData, error) { + keys := n.keyRotator.GetKeys() -func migrateWALs(legacyWALDir, walDir string) error { - // keep temporary wal directory so WAL initialization appears atomic - tmpdirpath := filepath.Clean(walDir) + ".tmp" - if fileutil.Exist(tmpdirpath) { - if err := os.RemoveAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not remove temporary WAL directory") - } - } - if err := fileutil.CreateDirAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not create temporary WAL directory") + n.raftLogger = &storage.EncryptedRaftLogger{ + StateDir: n.opts.StateDir, + EncryptionKey: keys.CurrentDEK, } - - walNames, err := fileutil.ReadDir(legacyWALDir) - if err != nil { - return errors.Wrapf(err, "could not list WAL directory %s", legacyWALDir) + if keys.PendingDEK != nil { + n.raftLogger.EncryptionKey = keys.PendingDEK } - for _, fname := range walNames { - _, err := copyFile(filepath.Join(legacyWALDir, fname), filepath.Join(tmpdirpath, fname), 0600) - if err != nil { - return errors.Wrap(err, "error copying WAL file") - } - } + snap, walData, err := n.raftLogger.BootstrapFromDisk(ctx) - if err := os.Rename(tmpdirpath, walDir); err != nil { - return err - } - - return nil -} - -func migrateSnapshots(legacySnapDir, snapDir string) error { - // use temporary snaphot directory so initialization appears atomic - tmpdirpath := filepath.Clean(snapDir) + ".tmp" - if fileutil.Exist(tmpdirpath) { - if err := os.RemoveAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not remove temporary snapshot directory") - } - } - if err := fileutil.CreateDirAll(tmpdirpath); err != nil { - return errors.Wrap(err, "could not create temporary snapshot directory") - } - - snapshotNames, err := fileutil.ReadDir(legacySnapDir) - if err != nil { - return errors.Wrapf(err, "could not list snapshot directory %s", legacySnapDir) - } - - for _, fname := range snapshotNames { - err := os.Link(filepath.Join(legacySnapDir, fname), filepath.Join(tmpdirpath, fname)) - if err != nil { - return errors.Wrap(err, "error linking snapshot file") + if keys.PendingDEK != nil { + switch errors.Cause(err).(type) { + case nil: + if err = n.keyRotator.UpdateKeys(EncryptionKeys{CurrentDEK: keys.PendingDEK}); err != nil { + err = errors.Wrap(err, "previous key rotation was successful, but unable mark rotation as complete") + } + case encryption.ErrCannotDecrypt: + snap, walData, err = n.raftLogger.BootstrapFromDisk(ctx, keys.CurrentDEK) } } - if err := os.Rename(tmpdirpath, snapDir); err != nil { - return err - } - - return nil -} - -// copyFile copies from src to dst until either EOF is reached -// on src or an error occurs. It verifies src exists and removes -// the dst if it exists. -func copyFile(src, dst string, perm os.FileMode) (int64, error) { - cleanSrc := filepath.Clean(src) - cleanDst := filepath.Clean(dst) - if cleanSrc == cleanDst { - return 0, nil - } - sf, err := os.Open(cleanSrc) - if err != nil { - return 0, err - } - defer sf.Close() - if err := os.Remove(cleanDst); err != nil && !os.IsNotExist(err) { - return 0, err - } - df, err := os.OpenFile(cleanDst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm) if err != nil { - return 0, err + return nil, storage.WALData{}, err } - defer df.Close() - return io.Copy(df, sf) + return snap, walData, nil } -func (n *Node) createWAL(nodeID string) (raft.Peer, error) { - raftNode := &api.RaftMember{ - RaftID: n.Config.ID, - NodeID: nodeID, - Addr: n.opts.Addr, - } - metadata, err := raftNode.Marshal() - if err != nil { - return raft.Peer{}, errors.Wrap(err, "error marshalling raft node") - } - n.wal, err = wal.Create(n.walDir(), metadata) - if err != nil { - return raft.Peer{}, errors.Wrap(err, "failed to create WAL") - } - - n.cluster.AddMember(&membership.Member{RaftMember: raftNode}) - return raft.Peer{ID: n.Config.ID, Context: metadata}, nil -} - -// moveWALAndSnap moves away the WAL and snapshot because we were removed -// from the cluster and will need to recreate them if we are readded. -func (n *Node) moveWALAndSnap() error { - newWALDir, err := ioutil.TempDir(n.opts.StateDir, "wal.") - if err != nil { - return err - } - err = os.Rename(n.walDir(), newWALDir) - if err != nil { - return err - } - - newSnapDir, err := ioutil.TempDir(n.opts.StateDir, "snap.") - if err != nil { - return err - } - err = os.Rename(n.snapDir(), newSnapDir) +// bootstraps a node's raft store from the raft logs and snapshots on disk +func (n *Node) loadAndStart(ctx context.Context, forceNewCluster bool) error { + snapshot, waldata, err := n.readFromDisk(ctx) if err != nil { return err } - return nil -} - -func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewCluster bool) (err error) { - var ( - walsnap walpb.Snapshot - metadata []byte - st raftpb.HardState - ents []raftpb.Entry - ) - if snapshot != nil { - walsnap.Index = snapshot.Metadata.Index - walsnap.Term = snapshot.Metadata.Term - } - - repaired := false - for { - if n.wal, err = wal.Open(n.walDir(), walsnap); err != nil { - return errors.Wrap(err, "failed to open WAL") - } - if metadata, st, ents, err = n.wal.ReadAll(); err != nil { - if err := n.wal.Close(); err != nil { - return err - } - // we can only repair ErrUnexpectedEOF and we never repair twice. - if repaired || err != io.ErrUnexpectedEOF { - return errors.Wrap(err, "irreparable WAL error") - } - if !wal.Repair(n.walDir()) { - return errors.Wrap(err, "WAL error cannot be repaired") - } - log.G(ctx).WithError(err).Info("repaired WAL error") - repaired = true - continue + // Load the snapshot data into the store + if err := n.restoreFromSnapshot(snapshot.Data, forceNewCluster); err != nil { + return err } - break } - defer func() { - if err != nil { - if walErr := n.wal.Close(); walErr != nil { - log.G(ctx).WithError(walErr).Error("error closing raft WAL") - } - } - }() - + // Read logs to fully catch up store var raftNode api.RaftMember - if err := raftNode.Unmarshal(metadata); err != nil { + if err := raftNode.Unmarshal(waldata.Metadata); err != nil { return errors.Wrap(err, "failed to unmarshal WAL metadata") } n.Config.ID = raftNode.RaftID + ents, st := waldata.Entries, waldata.HardState + // All members that are no longer part of the cluster must be added to // the removed list right away, so that we don't try to connect to them // before processing the configuration change entries, which could make @@ -326,7 +116,7 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC ents = append(ents, toAppEnts...) // force commit newly appended entries - err := n.wal.Save(st, toAppEnts) + err := n.raftLogger.SaveEntries(st, toAppEnts) if err != nil { log.G(ctx).WithError(err).Fatalf("failed to save WAL while forcing new cluster") } @@ -343,146 +133,24 @@ func (n *Node) readWAL(ctx context.Context, snapshot *raftpb.Snapshot, forceNewC if err := n.raftStore.SetHardState(st); err != nil { return err } - if err := n.raftStore.Append(ents); err != nil { - return err - } - - return nil + return n.raftStore.Append(ents) } -func (n *Node) saveSnapshot(snapshot raftpb.Snapshot, keepOldSnapshots uint64) error { - err := n.wal.SaveSnapshot(walpb.Snapshot{ - Index: snapshot.Metadata.Index, - Term: snapshot.Metadata.Term, - }) - if err != nil { - return err - } - err = n.snapshotter.SaveSnap(snapshot) - if err != nil { - return err - } - err = n.wal.ReleaseLockTo(snapshot.Metadata.Index) - if err != nil { - return err - } - - // Delete any older snapshots - curSnapshot := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, ".snap") - - dirents, err := ioutil.ReadDir(n.snapDir()) - if err != nil { - return err - } - - var snapshots []string - for _, dirent := range dirents { - if strings.HasSuffix(dirent.Name(), ".snap") { - snapshots = append(snapshots, dirent.Name()) - } - } - - // Sort snapshot filenames in reverse lexical order - sort.Sort(sort.Reverse(sort.StringSlice(snapshots))) - - // Ignore any snapshots that are older than the current snapshot. - // Delete the others. Rather than doing lexical comparisons, we look - // at what exists before/after the current snapshot in the slice. - // This means that if the current snapshot doesn't appear in the - // directory for some strange reason, we won't delete anything, which - // is the safe behavior. - curSnapshotIdx := -1 - var ( - removeErr error - oldestSnapshot string - ) - - for i, snapFile := range snapshots { - if curSnapshotIdx >= 0 && i > curSnapshotIdx { - if uint64(i-curSnapshotIdx) > keepOldSnapshots { - err := os.Remove(filepath.Join(n.snapDir(), snapFile)) - if err != nil && removeErr == nil { - removeErr = err - } - continue - } - } else if snapFile == curSnapshot { - curSnapshotIdx = i - } - oldestSnapshot = snapFile - } - - if removeErr != nil { - return removeErr - } - - // Remove any WAL files that only contain data from before the oldest - // remaining snapshot. - - if oldestSnapshot == "" { - return nil - } - - // Parse index out of oldest snapshot's filename - var snapTerm, snapIndex uint64 - _, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex) - if err != nil { - return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot) +func (n *Node) newRaftLogs(nodeID string) (raft.Peer, error) { + raftNode := &api.RaftMember{ + RaftID: n.Config.ID, + NodeID: nodeID, + Addr: n.opts.Addr, } - - // List the WALs - dirents, err = ioutil.ReadDir(n.walDir()) + metadata, err := raftNode.Marshal() if err != nil { - return err - } - - var wals []string - for _, dirent := range dirents { - if strings.HasSuffix(dirent.Name(), ".wal") { - wals = append(wals, dirent.Name()) - } - } - - // Sort WAL filenames in lexical order - sort.Sort(sort.StringSlice(wals)) - - found := false - deleteUntil := -1 - - for i, walName := range wals { - var walSeq, walIndex uint64 - _, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex) - if err != nil { - return errors.Wrapf(err, "could not parse WAL name %s", walName) - } - - if walIndex >= snapIndex { - deleteUntil = i - 1 - found = true - break - } - } - - // If all WAL files started with indices below the oldest snapshot's - // index, we can delete all but the newest WAL file. - if !found && len(wals) != 0 { - deleteUntil = len(wals) - 1 + return raft.Peer{}, errors.Wrap(err, "error marshalling raft node") } - - for i := 0; i < deleteUntil; i++ { - walPath := filepath.Join(n.walDir(), wals[i]) - l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode) - if err != nil { - return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i]) - } - err = os.Remove(walPath) - l.Close() - if err != nil { - return errors.Wrapf(err, "error removing old WAL file %s", wals[i]) - } + if err := n.raftLogger.BootstrapNew(metadata); err != nil { + return raft.Peer{}, err } - - return nil + n.cluster.AddMember(&membership.Member{RaftMember: raftNode}) + return raft.Peer{ID: n.Config.ID, Context: metadata}, nil } func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { @@ -497,6 +165,17 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { } snapshot.Membership.Removed = n.cluster.Removed() + // maybe start rotation + n.rotationQueued = false + var newEncryptionKeys *EncryptionKeys + if n.keyRotator.NeedsRotation() { + keys := n.keyRotator.GetKeys() + if keys.PendingDEK != nil { + n.raftLogger.RotateEncryptionKey(keys.PendingDEK) + newEncryptionKeys = &EncryptionKeys{CurrentDEK: keys.PendingDEK} + } + } + viewStarted := make(chan struct{}) n.asyncTasks.Add(1) n.snapshotInProgress = make(chan uint64, 1) // buffered in case Shutdown is called during the snapshot @@ -505,7 +184,6 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { n.asyncTasks.Done() n.snapshotInProgress <- snapshotIndex }() - var err error n.memoryStore.View(func(tx store.ReadTx) { close(viewStarted) @@ -526,11 +204,18 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { } snap, err := n.raftStore.CreateSnapshot(appliedIndex, &n.confState, d) if err == nil { - if err := n.saveSnapshot(snap, raftConfig.KeepOldSnapshots); err != nil { + if err := n.raftLogger.SaveSnapshot(snap); err != nil { log.G(ctx).WithError(err).Error("failed to save snapshot") return } snapshotIndex = appliedIndex + if newEncryptionKeys != nil { + // this means we tried to rotate - so finish the rotation + if err := n.keyRotator.UpdateKeys(*newEncryptionKeys); err != nil { + log.G(ctx).WithError(err).Error( + "failed to update encryption keys after a rotation - will wait for the next snapshot") + } + } if appliedIndex > raftConfig.LogEntriesForSlowFollowers { err := n.raftStore.Compact(appliedIndex - raftConfig.LogEntriesForSlowFollowers) @@ -538,6 +223,10 @@ func (n *Node) doSnapshot(ctx context.Context, raftConfig api.RaftConfig) { log.G(ctx).WithError(err).Error("failed to compact snapshot") } } + + if err := n.raftLogger.GC(snap.Metadata.Index, snap.Metadata.Term, raftConfig.KeepOldSnapshots); err != nil { + log.G(ctx).WithError(err).Error("failed to clean up old snapshots and WALs") + } } else if err != raft.ErrSnapOutOfDate { log.G(ctx).WithError(err).Error("failed to create snapshot") } diff --git a/manager/state/raft/storage/snapwrap.go b/manager/state/raft/storage/snapwrap.go index 08f475fb5f..05acbab4f0 100644 --- a/manager/state/raft/storage/snapwrap.go +++ b/manager/state/raft/storage/snapwrap.go @@ -1,9 +1,17 @@ package storage import ( + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/raft/raftpb" "github.com/coreos/etcd/snap" "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" ) // This package wraps the github.com/coreos/etcd/snap package, and encrypts @@ -57,6 +65,7 @@ func (s *wrappedSnap) Load() (*raftpb.Snapshot, error) { if err != nil { return nil, err } + return snapshot, nil } @@ -93,3 +102,57 @@ func (o originalSnap) New(dirpath string) Snapshotter { // OriginalSnap is the original `snap` package as an implemntation of the SnapFactory interface var OriginalSnap SnapFactory = originalSnap{} + +// MigrateSnapshot reads the latest existing snapshot from one directory, encoded one way, and writes +// it to a new directory, encoded a different way +func MigrateSnapshot(oldDir, newDir string, oldFactory, newFactory SnapFactory) error { + // use temporary snapshot directory so initialization appears atomic + oldSnapshotter := oldFactory.New(oldDir) + snapshot, err := oldSnapshotter.Load() + switch err { + case snap.ErrNoSnapshot: // if there's no snapshot, the migration succeeded + return nil + case nil: + break + default: + return err + } + + tmpdirpath := filepath.Clean(newDir) + ".tmp" + if fileutil.Exist(tmpdirpath) { + if err := os.RemoveAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not remove temporary snapshot directory") + } + } + if err := fileutil.CreateDirAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not create temporary snapshot directory") + } + tmpSnapshotter := newFactory.New(tmpdirpath) + + // write the new snapshot to the temporary location + if err = tmpSnapshotter.SaveSnap(*snapshot); err != nil { + return err + } + + return os.Rename(tmpdirpath, newDir) +} + +// ListSnapshots lists all the snapshot files in a particular directory and returns +// the snapshot files in reverse lexical order (newest first) +func ListSnapshots(dirpath string) ([]string, error) { + dirents, err := ioutil.ReadDir(dirpath) + if err != nil { + return nil, err + } + + var snapshots []string + for _, dirent := range dirents { + if strings.HasSuffix(dirent.Name(), ".snap") { + snapshots = append(snapshots, dirent.Name()) + } + } + + // Sort snapshot filenames in reverse lexical order + sort.Sort(sort.Reverse(sort.StringSlice(snapshots))) + return snapshots, nil +} diff --git a/manager/state/raft/storage/snapwrap_test.go b/manager/state/raft/storage/snapwrap_test.go index ebee634080..01e10ed4d4 100644 --- a/manager/state/raft/storage/snapwrap_test.go +++ b/manager/state/raft/storage/snapwrap_test.go @@ -1,6 +1,7 @@ package storage import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -174,3 +175,59 @@ func TestSaveAndLoad(t *testing.T) { require.NoError(t, err) require.Equal(t, fakeSnapshotData, *readSnap) } + +func TestMigrateSnapshot(t *testing.T) { + crypter := &meowCrypter{} + c := NewSnapFactory(crypter, crypter) + var ( + err error + dirs = make([]string, 3) + ) + + tempDir, err := ioutil.TempDir("", "test-migrate") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + for i := range dirs { + dirs[i] = filepath.Join(tempDir, fmt.Sprintf("snapDir%d", i)) + } + require.NoError(t, os.Mkdir(dirs[0], 0755)) + require.NoError(t, OriginalSnap.New(dirs[0]).SaveSnap(fakeSnapshotData)) + + // original to new + oldDir := dirs[0] + newDir := dirs[1] + + err = MigrateSnapshot(oldDir, newDir, OriginalSnap, c) + require.NoError(t, err) + + readSnap, err := c.New(newDir).Load() + require.NoError(t, err) + require.Equal(t, fakeSnapshotData, *readSnap) + + // new to original + oldDir = dirs[1] + newDir = dirs[2] + + err = MigrateSnapshot(oldDir, newDir, c, OriginalSnap) + require.NoError(t, err) + + readSnap, err = OriginalSnap.New(newDir).Load() + require.NoError(t, err) + require.Equal(t, fakeSnapshotData, *readSnap) + + // We can migrate from empty directory without error + for _, dir := range dirs { + require.NoError(t, os.RemoveAll(dir)) + } + require.NoError(t, os.Mkdir(dirs[0], 0755)) + oldDir = dirs[0] + newDir = dirs[1] + + err = MigrateSnapshot(oldDir, newDir, OriginalSnap, c) + require.NoError(t, err) + + subdirs, err := ioutil.ReadDir(tempDir) + require.NoError(t, err) + require.Len(t, subdirs, 1) +} diff --git a/manager/state/raft/storage/storage.go b/manager/state/raft/storage/storage.go new file mode 100644 index 0000000000..d830767ded --- /dev/null +++ b/manager/state/raft/storage/storage.go @@ -0,0 +1,391 @@ +package storage + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "golang.org/x/net/context" + + "github.com/coreos/etcd/pkg/fileutil" + "github.com/coreos/etcd/raft/raftpb" + "github.com/coreos/etcd/snap" + "github.com/coreos/etcd/wal" + "github.com/coreos/etcd/wal/walpb" + "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/log" + "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" +) + +// ErrNoWAL is returned if there are no WALs on disk +var ErrNoWAL = errors.New("no WAL present") + +type walSnapDirs struct { + wal string + snap string +} + +// the wal/snap directories in decreasing order of preference/version +var versionedWALSnapDirs = []walSnapDirs{ + {wal: "wal-v3-encrypted", snap: "snap-v3-encrypted"}, + {wal: "wal-v3", snap: "snap-v3"}, + {wal: "wal", snap: "snap"}, +} + +// MultiDecrypter attempts to decrypt with a list of decrypters +type MultiDecrypter []encryption.Decrypter + +// Decrypt tries to decrypt using all the decrypters +func (m MultiDecrypter) Decrypt(r api.MaybeEncryptedRecord) (result []byte, err error) { + for _, d := range m { + result, err = d.Decrypt(r) + if err == nil { + return + } + } + return +} + +// EncryptedRaftLogger saves raft data to disk +type EncryptedRaftLogger struct { + StateDir string + EncryptionKey []byte + + // mutex is locked for writing only when we need to replace the wal object and snapshotter + // object, not when we're writing snapshots or wals (in which case it's locked for reading) + encoderMu sync.RWMutex + wal WAL + snapshotter Snapshotter +} + +// BootstrapFromDisk creates a new snapshotter and wal, and also reads the latest snapshot and WALs from disk +func (e *EncryptedRaftLogger) BootstrapFromDisk(ctx context.Context, oldEncryptionKeys ...[]byte) (*raftpb.Snapshot, WALData, error) { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + walDir := e.walDir() + snapDir := e.snapDir() + + encrypter, decrypter := encryption.Defaults(e.EncryptionKey) + if oldEncryptionKeys != nil { + decrypters := []encryption.Decrypter{decrypter} + for _, key := range oldEncryptionKeys { + _, d := encryption.Defaults(key) + decrypters = append(decrypters, d) + } + decrypter = MultiDecrypter(decrypters) + } + + snapFactory := NewSnapFactory(encrypter, decrypter) + + if !fileutil.Exist(snapDir) { + // If snapshots created by the etcd-v2 code exist, or by swarmkit development version, + // read the latest snapshot and write it encoded to the new path. The new path + // prevents etc-v2 creating snapshots that are visible to us, but not encoded and + // out of sync with our WALs, after a downgrade. + for _, dirs := range versionedWALSnapDirs[1:] { + legacySnapDir := filepath.Join(e.StateDir, dirs.snap) + if fileutil.Exist(legacySnapDir) { + if err := MigrateSnapshot(legacySnapDir, snapDir, OriginalSnap, snapFactory); err != nil { + return nil, WALData{}, err + } + break + } + } + } + // ensure the new directory exists + if err := os.MkdirAll(snapDir, 0700); err != nil { + return nil, WALData{}, errors.Wrap(err, "failed to create snapshot directory") + } + + var ( + snapshotter Snapshotter + walObj WAL + err error + ) + + // Create a snapshotter and load snapshot data + snapshotter = snapFactory.New(snapDir) + snapshot, err := snapshotter.Load() + if err != nil && err != snap.ErrNoSnapshot { + return nil, WALData{}, err + } + + walFactory := NewWALFactory(encrypter, decrypter) + var walsnap walpb.Snapshot + if snapshot != nil { + walsnap.Index = snapshot.Metadata.Index + walsnap.Term = snapshot.Metadata.Term + } + + if !wal.Exist(walDir) { + var walExists bool + // If wals created by the etcd-v2 wal code exist, read the latest ones based + // on this snapshot and encode them to wals in the new path to avoid adding + // backwards-incompatible entries to those files. + for _, dirs := range versionedWALSnapDirs[1:] { + legacyWALDir := filepath.Join(e.StateDir, dirs.wal) + if !wal.Exist(legacyWALDir) { + continue + } + if err = MigrateWALs(ctx, legacyWALDir, walDir, OriginalWAL, walFactory, walsnap); err != nil { + return nil, WALData{}, err + } + walExists = true + break + } + if !walExists { + return nil, WALData{}, ErrNoWAL + } + } + + walObj, waldata, err := ReadRepairWAL(ctx, walDir, walsnap, walFactory) + if err != nil { + return nil, WALData{}, err + } + + e.snapshotter = snapshotter + e.wal = walObj + + return snapshot, waldata, nil +} + +// BootstrapNew creates a new snapshotter and WAL writer, expecting that there is nothing on disk +func (e *EncryptedRaftLogger) BootstrapNew(metadata []byte) error { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + encrypter, decrypter := encryption.Defaults(e.EncryptionKey) + walFactory := NewWALFactory(encrypter, decrypter) + + for _, dirpath := range []string{e.walDir(), e.snapDir()} { + if err := os.MkdirAll(dirpath, 0700); err != nil { + return errors.Wrapf(err, "failed to create %s", dirpath) + } + } + var err error + e.wal, err = walFactory.Create(e.walDir(), metadata) + if err != nil { + return errors.Wrap(err, "failed to create WAL") + } + + e.snapshotter = NewSnapFactory(encrypter, decrypter).New(e.snapDir()) + return nil +} + +func (e *EncryptedRaftLogger) walDir() string { + return filepath.Join(e.StateDir, versionedWALSnapDirs[0].wal) +} + +func (e *EncryptedRaftLogger) snapDir() string { + return filepath.Join(e.StateDir, versionedWALSnapDirs[0].snap) +} + +// RotateEncryptionKey swaps out the encoders and decoders used by the wal and snapshotter +func (e *EncryptedRaftLogger) RotateEncryptionKey(newKey []byte) { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + if e.wal != nil { // if the wal exists, the snapshotter exists + // We don't want to have to close the WAL, because we can't open a new one. + // We need to know the previous snapshot, because when you open a WAL you + // have to read out all the entries from a particular snapshot, or you can't + // write. So just rotate the encoders out from under it. We already + // have a lock on writing to snapshots and WALs. + wrapped, ok := e.wal.(*wrappedWAL) + if !ok { + panic(fmt.Errorf("EncryptedRaftLogger's WAL is not a wrappedWAL")) + } + + wrapped.encrypter, wrapped.decrypter = encryption.Defaults(newKey) + + e.snapshotter = NewSnapFactory(wrapped.encrypter, wrapped.decrypter).New(e.snapDir()) + } + e.EncryptionKey = newKey +} + +// SaveSnapshot actually saves a given snapshot to both the WAL and the snapshot. +func (e *EncryptedRaftLogger) SaveSnapshot(snapshot raftpb.Snapshot) error { + + walsnap := walpb.Snapshot{ + Index: snapshot.Metadata.Index, + Term: snapshot.Metadata.Term, + } + + e.encoderMu.RLock() + if err := e.wal.SaveSnapshot(walsnap); err != nil { + e.encoderMu.RUnlock() + return err + } + + snapshotter := e.snapshotter + e.encoderMu.RUnlock() + + if err := snapshotter.SaveSnap(snapshot); err != nil { + return err + } + if err := e.wal.ReleaseLockTo(snapshot.Metadata.Index); err != nil { + return err + } + return nil +} + +// GC garbage collects snapshots and wals older than the provided index and term +func (e *EncryptedRaftLogger) GC(index uint64, term uint64, keepOldSnapshots uint64) error { + // Delete any older snapshots + curSnapshot := fmt.Sprintf("%016x-%016x%s", term, index, ".snap") + + snapshots, err := ListSnapshots(e.snapDir()) + if err != nil { + return err + } + + // Ignore any snapshots that are older than the current snapshot. + // Delete the others. Rather than doing lexical comparisons, we look + // at what exists before/after the current snapshot in the slice. + // This means that if the current snapshot doesn't appear in the + // directory for some strange reason, we won't delete anything, which + // is the safe behavior. + curSnapshotIdx := -1 + var ( + removeErr error + oldestSnapshot string + ) + + for i, snapFile := range snapshots { + if curSnapshotIdx >= 0 && i > curSnapshotIdx { + if uint64(i-curSnapshotIdx) > keepOldSnapshots { + err := os.Remove(filepath.Join(e.snapDir(), snapFile)) + if err != nil && removeErr == nil { + removeErr = err + } + continue + } + } else if snapFile == curSnapshot { + curSnapshotIdx = i + } + oldestSnapshot = snapFile + } + + if removeErr != nil { + return removeErr + } + + // Remove any WAL files that only contain data from before the oldest + // remaining snapshot. + + if oldestSnapshot == "" { + return nil + } + + // Parse index out of oldest snapshot's filename + var snapTerm, snapIndex uint64 + _, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex) + if err != nil { + return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot) + } + + wals, err := ListWALs(e.walDir()) + if err != nil { + return err + } + + found := false + deleteUntil := -1 + + for i, walName := range wals { + var walSeq, walIndex uint64 + _, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex) + if err != nil { + return errors.Wrapf(err, "could not parse WAL name %s", walName) + } + + if walIndex >= snapIndex { + deleteUntil = i - 1 + found = true + break + } + } + + // If all WAL files started with indices below the oldest snapshot's + // index, we can delete all but the newest WAL file. + if !found && len(wals) != 0 { + deleteUntil = len(wals) - 1 + } + + for i := 0; i < deleteUntil; i++ { + walPath := filepath.Join(e.walDir(), wals[i]) + l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode) + if err != nil { + return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i]) + } + err = os.Remove(walPath) + l.Close() + if err != nil { + return errors.Wrapf(err, "error removing old WAL file %s", wals[i]) + } + } + + return nil +} + +// SaveEntries saves only entries to disk +func (e *EncryptedRaftLogger) SaveEntries(st raftpb.HardState, entries []raftpb.Entry) error { + e.encoderMu.RLock() + defer e.encoderMu.RUnlock() + + if e.wal == nil { + return fmt.Errorf("raft WAL has either been closed or has never been created") + } + return e.wal.Save(st, entries) +} + +// Close closes the logger - it will have to be bootstrapped again to start writing +func (e *EncryptedRaftLogger) Close(ctx context.Context) { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + if e.wal != nil { + if err := e.wal.Close(); err != nil { + log.G(ctx).WithError(err).Error("error closing raft WAL") + } + } + + e.wal = nil + e.snapshotter = nil +} + +// Clear closes the existing WAL and moves away the WAL and snapshot. +func (e *EncryptedRaftLogger) Clear(ctx context.Context) error { + e.encoderMu.Lock() + defer e.encoderMu.Unlock() + + if e.wal != nil { + if err := e.wal.Close(); err != nil { + log.G(ctx).WithError(err).Error("error closing raft WAL") + } + } + e.snapshotter = nil + + newWALDir, err := ioutil.TempDir(e.StateDir, "wal.") + if err != nil { + return err + } + err = os.Rename(e.walDir(), newWALDir) + if err != nil { + return err + } + + newSnapDir, err := ioutil.TempDir(e.StateDir, "snap.") + if err != nil { + return err + } + err = os.Rename(e.snapDir(), newSnapDir) + if err != nil { + return err + } + + return nil +} diff --git a/manager/state/raft/storage/storage_test.go b/manager/state/raft/storage/storage_test.go new file mode 100644 index 0000000000..2683d1d735 --- /dev/null +++ b/manager/state/raft/storage/storage_test.go @@ -0,0 +1,221 @@ +package storage + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/coreos/etcd/raft/raftpb" + "github.com/coreos/etcd/wal/walpb" + "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestBootstrapFromDisk(t *testing.T) { + tempdir, err := ioutil.TempDir("", "raft-storage") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key1"), + } + err = logger.BootstrapNew([]byte("metadata")) + require.NoError(t, err) + + // everything should be saved with "key1" + _, entries, _ := makeWALData(0, 0) + err = logger.SaveEntries(raftpb.HardState{}, entries) + require.NoError(t, err) + logger.Close(context.Background()) + + // now we can bootstrap from disk, even if there is no snapshot + logger = EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key1"), + } + readSnap, waldata, err := logger.BootstrapFromDisk(context.Background()) + require.NoError(t, err) + require.Nil(t, readSnap) + require.Equal(t, entries, waldata.Entries) + + // save a snapshot + snapshot := fakeSnapshotData + err = logger.SaveSnapshot(snapshot) + require.NoError(t, err) + _, entries, _ = makeWALData(snapshot.Metadata.Index, snapshot.Metadata.Term) + err = logger.SaveEntries(raftpb.HardState{}, entries) + require.NoError(t, err) + logger.Close(context.Background()) + + // load snapshots and wals + logger = EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key1"), + } + readSnap, waldata, err = logger.BootstrapFromDisk(context.Background()) + require.NoError(t, err) + require.NotNil(t, snapshot) + require.Equal(t, snapshot, *readSnap) + require.Equal(t, entries, waldata.Entries) + + // start writing more wals and rotate in the middle + _, entries, _ = makeWALData(snapshot.Metadata.Index, snapshot.Metadata.Term) + err = logger.SaveEntries(raftpb.HardState{}, entries[:1]) + require.NoError(t, err) + logger.RotateEncryptionKey([]byte("key2")) + err = logger.SaveEntries(raftpb.HardState{}, entries[1:]) + require.NoError(t, err) + logger.Close(context.Background()) + + // we can't bootstrap from disk using only the first or second key + for _, key := range []string{"key1", "key2"} { + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte(key), + } + _, _, err := logger.BootstrapFromDisk(context.Background()) + require.IsType(t, encryption.ErrCannotDecrypt{}, errors.Cause(err)) + } + + // but we can if we combine the two keys, we can bootstrap just fine + logger = EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("key2"), + } + readSnap, waldata, err = logger.BootstrapFromDisk(context.Background(), []byte("key1")) + require.NoError(t, err) + require.NotNil(t, snapshot) + require.Equal(t, snapshot, *readSnap) + require.Equal(t, entries, waldata.Entries) +} + +// Ensure that we can change encoding and not have a race condition +func TestRaftLoggerRace(t *testing.T) { + tempdir, err := ioutil.TempDir("", "raft-storage") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: []byte("Hello"), + } + err = logger.BootstrapNew([]byte("metadata")) + require.NoError(t, err) + + _, entries, _ := makeWALData(fakeSnapshotData.Metadata.Index, fakeSnapshotData.Metadata.Term) + + done1 := make(chan error) + done2 := make(chan error) + done3 := make(chan error) + done4 := make(chan error) + go func() { + done1 <- logger.SaveSnapshot(fakeSnapshotData) + }() + go func() { + done2 <- logger.SaveEntries(raftpb.HardState{}, entries) + }() + go func() { + logger.RotateEncryptionKey([]byte("Hello 2")) + done3 <- nil + }() + go func() { + done4 <- logger.SaveSnapshot(fakeSnapshotData) + }() + + err = <-done1 + require.NoError(t, err, "unable to save snapshot") + + err = <-done2 + require.NoError(t, err, "unable to save entries") + + err = <-done3 + require.NoError(t, err, "unable to rotate key") + + err = <-done4 + require.NoError(t, err, "unable to save snapshot a second time") +} + +func TestMigrateToV3EncryptedForm(t *testing.T) { + t.Parallel() + + tempdir, err := ioutil.TempDir("", "raft-storage") + require.NoError(t, err) + defer os.RemoveAll(tempdir) + + dek := []byte("key") + + writeDataTo := func(suffix string, snapshot raftpb.Snapshot, walFactory WALFactory, snapFactory SnapFactory) []raftpb.Entry { + snapDir := filepath.Join(tempdir, "snap"+suffix) + walDir := filepath.Join(tempdir, "wal"+suffix) + for _, dirpath := range []string{snapDir, walDir} { + require.NoError(t, os.MkdirAll(dirpath, 0755)) + } + require.NoError(t, snapFactory.New(snapDir).SaveSnap(snapshot)) + + _, entries, _ := makeWALData(snapshot.Metadata.Index, snapshot.Metadata.Term) + walWriter, err := walFactory.Create(walDir, []byte("metadata")) + require.NoError(t, err) + require.NoError(t, walWriter.SaveSnapshot(walpb.Snapshot{Index: snapshot.Metadata.Index, Term: snapshot.Metadata.Term})) + require.NoError(t, walWriter.Save(raftpb.HardState{}, entries)) + require.NoError(t, walWriter.Close()) + return entries + } + + requireLoadedData := func(expectedSnap raftpb.Snapshot, expectedEntries []raftpb.Entry) { + logger := EncryptedRaftLogger{ + StateDir: tempdir, + EncryptionKey: dek, + } + readSnap, waldata, err := logger.BootstrapFromDisk(context.Background()) + require.NoError(t, err) + require.NotNil(t, readSnap) + require.Equal(t, expectedSnap, *readSnap) + require.Equal(t, expectedEntries, waldata.Entries) + logger.Close(context.Background()) + } + + v2Snapshot := fakeSnapshotData + v3Snapshot := fakeSnapshotData + v3Snapshot.Metadata.Index += 100 + v3Snapshot.Metadata.Term += 10 + v3EncryptedSnapshot := fakeSnapshotData + v3EncryptedSnapshot.Metadata.Index += 200 + v3EncryptedSnapshot.Metadata.Term += 20 + + encoder, decoders := encryption.Defaults(dek) + walFactory := NewWALFactory(encoder, decoders) + snapFactory := NewSnapFactory(encoder, decoders) + + // generate both v2 and v3 unencrypted snapshot data directories, as well as an encrypted directory + v2Entries := writeDataTo("", v2Snapshot, OriginalWAL, OriginalSnap) + v3Entries := writeDataTo("-v3", v3Snapshot, OriginalWAL, OriginalSnap) + v3EncryptedEntries := writeDataTo("-v3-encrypted", v3EncryptedSnapshot, walFactory, snapFactory) + + // bootstrap from disk - the encrypted directory exists, so we should just read from + // it instead of from the legacy directories + requireLoadedData(v3EncryptedSnapshot, v3EncryptedEntries) + + // remove the newest dirs - should try to migrate from v3 + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3-encrypted"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "wal-v3-encrypted"))) + requireLoadedData(v3Snapshot, v3Entries) + // it can recover from partial migrations + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3-encrypted"))) + requireLoadedData(v3Snapshot, v3Entries) + // v3 dirs still there + _, err = os.Stat(filepath.Join(tempdir, "snap-v3")) + require.NoError(t, err) + _, err = os.Stat(filepath.Join(tempdir, "wal-v3")) + require.NoError(t, err) + + // remove the v3 dirs - should try to migrate from v2 + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3-encrypted"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "wal-v3-encrypted"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "snap-v3"))) + require.NoError(t, os.RemoveAll(filepath.Join(tempdir, "wal-v3"))) + requireLoadedData(v2Snapshot, v2Entries) +} diff --git a/manager/state/raft/storage/walwrap.go b/manager/state/raft/storage/walwrap.go index fdd663262b..87009d4827 100644 --- a/manager/state/raft/storage/walwrap.go +++ b/manager/state/raft/storage/walwrap.go @@ -1,10 +1,21 @@ package storage import ( + "context" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/coreos/etcd/pkg/fileutil" "github.com/coreos/etcd/raft/raftpb" "github.com/coreos/etcd/wal" "github.com/coreos/etcd/wal/walpb" + "github.com/docker/swarmkit/log" "github.com/docker/swarmkit/manager/encryption" + "github.com/pkg/errors" ) // This package wraps the github.com/coreos/etcd/wal package, and encrypts @@ -128,3 +139,115 @@ func (o originalWAL) Open(dirpath string, walsnap walpb.Snapshot) (WAL, error) { // OriginalWAL is the original `wal` package as an implemntation of the WALFactory interface var OriginalWAL WALFactory = originalWAL{} + +// WALData contains all the data returned by a WAL's ReadAll() function +// (metadata, hardwate, and entries) +type WALData struct { + Metadata []byte + HardState raftpb.HardState + Entries []raftpb.Entry +} + +// ReadRepairWAL opens a WAL for reading, and attempts to read it. If we can't read it, attempts to repair +// and read again. +func ReadRepairWAL( + ctx context.Context, + walDir string, + walsnap walpb.Snapshot, + factory WALFactory, +) (WAL, WALData, error) { + var ( + reader WAL + metadata []byte + st raftpb.HardState + ents []raftpb.Entry + err error + ) + repaired := false + for { + if reader, err = factory.Open(walDir, walsnap); err != nil { + return nil, WALData{}, errors.Wrap(err, "failed to open WAL") + } + if metadata, st, ents, err = reader.ReadAll(); err != nil { + if closeErr := reader.Close(); closeErr != nil { + return nil, WALData{}, closeErr + } + if _, ok := err.(encryption.ErrCannotDecrypt); ok { + return nil, WALData{}, errors.Wrap(err, "failed to decrypt WAL") + } + // we can only repair ErrUnexpectedEOF and we never repair twice. + if repaired || err != io.ErrUnexpectedEOF { + return nil, WALData{}, errors.Wrap(err, "irreparable WAL error") + } + if !wal.Repair(walDir) { + return nil, WALData{}, errors.Wrap(err, "WAL error cannot be repaired") + } + log.G(ctx).WithError(err).Info("repaired WAL error") + repaired = true + continue + } + break + } + return reader, WALData{ + Metadata: metadata, + HardState: st, + Entries: ents, + }, nil +} + +// MigrateWALs reads existing WALs (from a particular snapshot and beyond) from one directory, encoded one way, +// and writes them to a new directory, encoded a different way +func MigrateWALs(ctx context.Context, oldDir, newDir string, oldFactory, newFactory WALFactory, snapshot walpb.Snapshot) error { + oldReader, waldata, err := ReadRepairWAL(ctx, oldDir, snapshot, oldFactory) + if err != nil { + return err + } + oldReader.Close() + + // keep temporary wal directory so WAL initialization appears atomic + tmpdirpath := filepath.Clean(newDir) + ".tmp" + if fileutil.Exist(tmpdirpath) { + if err := os.RemoveAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not remove temporary WAL directory") + } + } + if err := fileutil.CreateDirAll(tmpdirpath); err != nil { + return errors.Wrap(err, "could not create temporary WAL directory") + } + + tmpWAL, err := newFactory.Create(tmpdirpath, waldata.Metadata) + if err != nil { + return errors.Wrap(err, "could not create new WAL in temporary WAL directory") + } + defer tmpWAL.Close() + + if err := tmpWAL.SaveSnapshot(snapshot); err != nil { + return errors.Wrap(err, "could not write WAL snapshot in temporary directory") + } + + if err := tmpWAL.Save(waldata.HardState, waldata.Entries); err != nil { + return errors.Wrap(err, "could not migrate WALs to temporary directory") + } + + return os.Rename(tmpdirpath, newDir) +} + +// ListWALs lists all the wals in a directory and returns the list in lexical +// order (oldest first) +func ListWALs(dirpath string) ([]string, error) { + dirents, err := ioutil.ReadDir(dirpath) + if err != nil { + return nil, err + } + + var wals []string + for _, dirent := range dirents { + if strings.HasSuffix(dirent.Name(), ".wal") { + wals = append(wals, dirent.Name()) + } + } + + // Sort WAL filenames in lexical order + sort.Sort(sort.StringSlice(wals)) + return wals, nil +} diff --git a/manager/state/raft/storage/walwrap_test.go b/manager/state/raft/storage/walwrap_test.go index 6f93f22f94..24ca563eb5 100644 --- a/manager/state/raft/storage/walwrap_test.go +++ b/manager/state/raft/storage/walwrap_test.go @@ -2,9 +2,11 @@ package storage import ( "bytes" + "context" "fmt" "io/ioutil" "os" + "path/filepath" "testing" "github.com/coreos/etcd/raft/raftpb" @@ -17,20 +19,22 @@ import ( var _ WALFactory = walCryptor{} // Generates a bunch of WAL test data -func makeWALData() ([]byte, []raftpb.Entry, walpb.Snapshot) { - term := uint64(3) - index := uint64(4) +func makeWALData(index uint64, term uint64) ([]byte, []raftpb.Entry, walpb.Snapshot) { + wsn := walpb.Snapshot{ + Index: index, + Term: term, + } var entries []raftpb.Entry - for i := index + 1; i < index+6; i++ { + for i := wsn.Index + 1; i < wsn.Index+6; i++ { entries = append(entries, raftpb.Entry{ - Term: term, + Term: wsn.Term + 1, Index: i, Data: []byte(fmt.Sprintf("Entry %d", i)), }) } - return []byte("metadata"), entries, walpb.Snapshot{Index: index, Term: term} + return []byte("metadata"), entries, wsn } func createWithWAL(t *testing.T, w WALFactory, metadata []byte, startSnap walpb.Snapshot, entries []raftpb.Entry) string { @@ -49,7 +53,7 @@ func createWithWAL(t *testing.T, w WALFactory, metadata []byte, startSnap walpb. // WAL can read entries are not wrapped, but not encrypted func TestReadAllWrappedNoEncryption(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) wrappedEntries := make([]raftpb.Entry, len(entries)) for i, entry := range entries { r := api.MaybeEncryptedRecord{Data: entry.Data} @@ -77,7 +81,7 @@ func TestReadAllWrappedNoEncryption(t *testing.T) { // When reading WAL, if the decrypter can't read the encryption type, errors func TestReadAllNoSupportedDecrypter(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) for i, entry := range entries { r := api.MaybeEncryptedRecord{Data: entry.Data, Algorithm: api.MaybeEncryptedRecord_Algorithm(-3)} data, err := r.Marshal() @@ -102,7 +106,7 @@ func TestReadAllNoSupportedDecrypter(t *testing.T) { // entry is incorrectly encryptd, an error is returned func TestReadAllEntryIncorrectlyEncrypted(t *testing.T) { crypter := &meowCrypter{} - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) // metadata is correctly encryptd, but entries are not meow-encryptd for i, entry := range entries { @@ -128,7 +132,7 @@ func TestReadAllEntryIncorrectlyEncrypted(t *testing.T) { // The entry data and metadata are encryptd with the given encrypter, and a regular // WAL will see them as such. func TestSave(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) crypter := &meowCrypter{} c := NewWALFactory(crypter, encryption.NoopCrypter) @@ -154,7 +158,7 @@ func TestSave(t *testing.T) { // If encryption fails, saving will fail func TestSaveEncryptionFails(t *testing.T) { - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) tempdir, err := ioutil.TempDir("", "waltests") require.NoError(t, err) @@ -162,7 +166,7 @@ func TestSaveEncryptionFails(t *testing.T) { // fail encrypting one of the entries, but not the first one c := NewWALFactory(&meowCrypter{encryptFailures: map[string]struct{}{ - "Entry 7": {}, + "Entry 3": {}, }}, nil) wrapped, err := c.Create(tempdir, metadata) require.NoError(t, err) @@ -198,7 +202,7 @@ func TestCreateOpenInvalidDirFails(t *testing.T) { // A WAL can read what it wrote so long as it has a corresponding decrypter func TestSaveAndRead(t *testing.T) { crypter := &meowCrypter{} - metadata, entries, snapshot := makeWALData() + metadata, entries, snapshot := makeWALData(1, 1) c := NewWALFactory(crypter, crypter) tempdir := createWithWAL(t, c, metadata, snapshot, entries) @@ -213,3 +217,97 @@ func TestSaveAndRead(t *testing.T) { require.Equal(t, metadata, meta) require.Equal(t, entries, ents) } + +func TestReadRepairWAL(t *testing.T) { + metadata, entries, snapshot := makeWALData(1, 1) + tempdir := createWithWAL(t, OriginalWAL, metadata, snapshot, entries) + defer os.RemoveAll(tempdir) + + // there should only be one WAL file in there - corrupt it + files, err := ioutil.ReadDir(tempdir) + require.NoError(t, err) + require.Len(t, files, 1) + + fName := filepath.Join(tempdir, files[0].Name()) + fileContents, err := ioutil.ReadFile(fName) + require.NoError(t, err) + require.NoError(t, ioutil.WriteFile(fName, fileContents[:200], files[0].Mode())) + + ogWAL, err := OriginalWAL.Open(tempdir, snapshot) + require.NoError(t, err) + _, _, _, err = ogWAL.ReadAll() + require.Error(t, err) + require.NoError(t, ogWAL.Close()) + + ogWAL, waldata, err := ReadRepairWAL(context.Background(), tempdir, snapshot, OriginalWAL) + require.NoError(t, err) + require.Equal(t, metadata, waldata.Metadata) + require.NoError(t, ogWAL.Close()) +} + +func TestMigrateWALs(t *testing.T) { + metadata, entries, snapshot := makeWALData(1, 1) + coder := &meowCrypter{} + c := NewWALFactory(coder, coder) + + var ( + err error + dirs = make([]string, 2) + ) + + tempDir, err := ioutil.TempDir("", "test-migrate") + require.NoError(t, err) + defer os.RemoveAll(tempDir) + + for i := range dirs { + dirs[i] = filepath.Join(tempDir, fmt.Sprintf("walDir%d", i)) + } + + origDir := createWithWAL(t, OriginalWAL, metadata, snapshot, entries) + defer os.RemoveAll(origDir) + + // original to new + oldDir := origDir + newDir := dirs[0] + + err = MigrateWALs(context.Background(), oldDir, newDir, OriginalWAL, c, snapshot) + require.NoError(t, err) + + newWAL, err := c.Open(newDir, snapshot) + require.NoError(t, err) + meta, _, ents, err := newWAL.ReadAll() + require.NoError(t, err) + require.Equal(t, metadata, meta) + require.Equal(t, entries, ents) + require.NoError(t, newWAL.Close()) + + // new to original + oldDir = dirs[0] + newDir = dirs[1] + + err = MigrateWALs(context.Background(), oldDir, newDir, c, OriginalWAL, snapshot) + require.NoError(t, err) + + newWAL, err = OriginalWAL.Open(newDir, snapshot) + require.NoError(t, err) + meta, _, ents, err = newWAL.ReadAll() + require.NoError(t, err) + require.Equal(t, metadata, meta) + require.Equal(t, entries, ents) + require.NoError(t, newWAL.Close()) + + // If we can't read the old directory (for instance if it doesn't exist), a temp directory + // is not created + for _, dir := range dirs { + require.NoError(t, os.RemoveAll(dir)) + } + oldDir = dirs[0] + newDir = dirs[1] + + err = MigrateWALs(context.Background(), oldDir, newDir, OriginalWAL, c, walpb.Snapshot{}) + require.Error(t, err) + + subdirs, err := ioutil.ReadDir(tempDir) + require.NoError(t, err) + require.Empty(t, subdirs) +} diff --git a/manager/state/raft/storage_test.go b/manager/state/raft/storage_test.go index ba8af148cd..61052a2392 100644 --- a/manager/state/raft/storage_test.go +++ b/manager/state/raft/storage_test.go @@ -10,6 +10,8 @@ import ( "time" "github.com/docker/swarmkit/api" + "github.com/docker/swarmkit/manager/state/raft" + "github.com/docker/swarmkit/manager/state/raft/storage" raftutils "github.com/docker/swarmkit/manager/state/raft/testutils" "github.com/docker/swarmkit/manager/state/store" "github.com/pivotal-golang/clock/fakeclock" @@ -38,7 +40,7 @@ func TestRaftSnapshot(t *testing.T) { // None of the nodes should have snapshot files yet for _, node := range nodes { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) assert.NoError(t, err) assert.Len(t, dirents, 0) } @@ -57,7 +59,7 @@ func TestRaftSnapshot(t *testing.T) { // All nodes should now have a snapshot file for nodeID, node := range nodes { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -74,7 +76,7 @@ func TestRaftSnapshot(t *testing.T) { // It should get a copy of the snapshot assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[4].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[4].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -110,7 +112,7 @@ func TestRaftSnapshot(t *testing.T) { // All nodes should have a snapshot under a *different* name for nodeID, node := range nodes { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -118,7 +120,7 @@ func TestRaftSnapshot(t *testing.T) { return fmt.Errorf("expected 1 snapshot, found %d on node %d", len(dirents), nodeID) } if dirents[0].Name() == snapshotFilenames[nodeID] { - return fmt.Errorf("snapshot %s did not get replaced", snapshotFilenames[nodeID]) + return fmt.Errorf("snapshot %s did not get replaced on node %d", snapshotFilenames[nodeID], nodeID) } return nil })) @@ -155,7 +157,7 @@ func TestRaftSnapshotRestart(t *testing.T) { // Remaining nodes shouldn't have snapshot files yet for _, node := range []*raftutils.TestNode{nodes[1], nodes[2]} { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) assert.NoError(t, err) assert.Len(t, dirents, 0) } @@ -168,7 +170,7 @@ func TestRaftSnapshotRestart(t *testing.T) { // Remaining nodes should now have a snapshot file for nodeIdx, node := range []*raftutils.TestNode{nodes[1], nodes[2]} { assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(node.StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -190,7 +192,7 @@ func TestRaftSnapshotRestart(t *testing.T) { // New node should get a copy of the snapshot assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -200,7 +202,7 @@ func TestRaftSnapshotRestart(t *testing.T) { return nil })) - dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[5].StateDir, "snap-v3-encrypted")) assert.NoError(t, err) assert.Len(t, dirents, 1) raftutils.CheckValuesOnNodes(t, clockSource, map[uint64]*raftutils.TestNode{1: nodes[1], 2: nodes[2]}, nodeIDs[:5], values[:5]) @@ -271,7 +273,7 @@ func TestGCWAL(t *testing.T) { // Snapshot should have been triggered just as the WAL rotated, so // both WAL files should be preserved assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -279,7 +281,7 @@ func TestGCWAL(t *testing.T) { return fmt.Errorf("expected 1 snapshot, found %d", len(dirents)) } - dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3")) + dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted")) if err != nil { return err } @@ -311,7 +313,7 @@ func TestGCWAL(t *testing.T) { // This time only one WAL file should be saved. assert.NoError(t, raftutils.PollFunc(clockSource, func() error { - dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3")) + dirents, err := ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "snap-v3-encrypted")) if err != nil { return err } @@ -320,7 +322,7 @@ func TestGCWAL(t *testing.T) { return fmt.Errorf("expected 1 snapshot, found %d", len(dirents)) } - dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3")) + dirents, err = ioutil.ReadDir(filepath.Join(nodes[1].StateDir, "wal-v3-encrypted")) if err != nil { return err } @@ -437,23 +439,229 @@ func proposeLargeValue(t *testing.T, raftNode *raftutils.TestNode, time time.Dur return node, nil } -func TestMigrateWAL(t *testing.T) { +func TestRaftEncryptionKeyRotation(t *testing.T) { + t.Parallel() nodes := make(map[uint64]*raftutils.TestNode) var clockSource *fakeclock.FakeClock - nodes[1], clockSource = raftutils.NewInitNode(t, tc, nil) + raftConfig := raft.DefaultRaftConfig() + nodes[1], clockSource = raftutils.NewInitNode(t, tc, &raftConfig) defer raftutils.TeardownCluster(t, nodes) - value, err := raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, "id1") - assert.NoError(t, err, "failed to propose value") - raftutils.CheckValuesOnNodes(t, clockSource, nodes, []string{"id1"}, []*api.Node{value}) + nodeIDs := []string{"id1", "id2", "id3", "id4", "id5", "id6", "id7"} + values := make([]*api.Node, len(nodeIDs)) + + // Propose 3 values + var err error + for i, nodeID := range nodeIDs[:3] { + values[i], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeID) + require.NoError(t, err, "failed to propose value") + } + + snapDir := filepath.Join(nodes[1].StateDir, "snap-v3-encrypted") + + startingKeys := nodes[1].KeyRotator.GetKeys() + + // rotate the encryption key + nodes[1].KeyRotator.QueuePendingKey([]byte("key2")) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + // the rotation should trigger a snapshot, which should notify the rotator when it's done + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshot, found %d on new node", len(snapshots)) + } + if nodes[1].KeyRotator.NeedsRotation() { + return fmt.Errorf("rotation never finished") + } + return nil + })) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:3], values[:3]) + + // Propose a 4th value + values[3], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[3]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:4], values[:4]) nodes[1].Server.Stop() nodes[1].ShutdownRaft() - // Move WAL directory so it looks like it was created by an old version - require.NoError(t, os.Rename(filepath.Join(nodes[1].StateDir, "wal-v3"), filepath.Join(nodes[1].StateDir, "wal"))) + // Try to restart node 1. Without the new unlock key, it can't actually start + n, ctx := raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(startingKeys)) + require.Error(t, n.Node.JoinAndStart(ctx), + "should not have been able to restart since we can't read snapshots") + // with the right key, it can start, even if the right key is only the pending key + newKeys := startingKeys + newKeys.PendingDEK = []byte("key2") + nodes[1].KeyRotator = raftutils.NewSimpleKeyRotator(newKeys) nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], false) - raftutils.CheckValuesOnNodes(t, clockSource, nodes, []string{"id1"}, []*api.Node{value}) + + raftutils.WaitForCluster(t, clockSource, nodes) + + // as soon as we joined, it should have finished rotating the key + require.False(t, nodes[1].KeyRotator.NeedsRotation()) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:4], values[:4]) + + // break snapshotting, and ensure that key rotation never finishes + tempSnapDir := filepath.Join(nodes[1].StateDir, "snap-backup") + require.NoError(t, os.Rename(snapDir, tempSnapDir)) + require.NoError(t, ioutil.WriteFile(snapDir, []byte("this is no longer a directory"), 0644)) + + nodes[1].KeyRotator.QueuePendingKey([]byte("key3")) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + time.Sleep(250 * time.Millisecond) + + // rotation has not been finished, because we cannot take a snapshot + require.True(t, nodes[1].KeyRotator.NeedsRotation()) + + // Propose a 5th value, so we have WALs written with the new key + values[4], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[4]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:5], values[:5]) + + nodes[1].Server.Stop() + nodes[1].ShutdownRaft() + + // restore the snapshot dir + require.NoError(t, os.RemoveAll(snapDir)) + require.NoError(t, os.Rename(tempSnapDir, snapDir)) + + // Now the wals are a mix of key2 and key3 - we can't actually start with either key + singleKey := raft.EncryptionKeys{CurrentDEK: []byte("key2")} + n, ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(singleKey)) + require.Error(t, n.Node.JoinAndStart(ctx), + "should not have been able to restart since we can't read all the WALs, even if we can read the snapshot") + singleKey = raft.EncryptionKeys{CurrentDEK: []byte("key3")} + n, ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, raftutils.NewSimpleKeyRotator(singleKey)) + require.Error(t, n.Node.JoinAndStart(ctx), + "should not have been able to restart since we can't read all the WALs, and also not the snapshot") + + nodes[1], ctx = raftutils.CopyNode(t, clockSource, nodes[1], false, + raftutils.NewSimpleKeyRotator(raft.EncryptionKeys{ + CurrentDEK: []byte("key2"), + PendingDEK: []byte("key3"), + })) + require.NoError(t, nodes[1].Node.JoinAndStart(ctx)) + + // we can load, but we still need a snapshot because rotation hasn't finished + snapshots, err := storage.ListSnapshots(snapDir) + require.NoError(t, err) + require.Len(t, snapshots, 1, "expected 1 snapshot") + require.True(t, nodes[1].KeyRotator.NeedsRotation()) + currSnapshot := snapshots[0] + + // start the node - everything should fix itself + go nodes[1].Node.Run(ctx) + raftutils.WaitForCluster(t, clockSource, nodes) + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + if nodes[1].KeyRotator.NeedsRotation() { + return fmt.Errorf("rotation never finished") + } + currSnapshot = snapshots[0] + return nil + })) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:5], values[:5]) + + // If we can't update the keys, we wait for the next snapshot to do so + nodes[1].KeyRotator.SetUpdateFunc(func() error { return fmt.Errorf("nope!") }) + nodes[1].KeyRotator.QueuePendingKey([]byte("key4")) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + currSnapshot = snapshots[0] + return nil + })) + require.True(t, nodes[1].KeyRotator.NeedsRotation()) + + // Fix updating the key rotator, and propose a 6th value, so another snapshot is + // triggered. + nodes[1].KeyRotator.SetUpdateFunc(nil) + values[5], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[5]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:6], values[:6]) + raftutils.AdvanceTicks(clockSource, 5) + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + if nodes[1].KeyRotator.NeedsRotation() { + return fmt.Errorf("rotation never finished") + } + currSnapshot = snapshots[0] + return nil + })) + require.False(t, nodes[1].KeyRotator.NeedsRotation()) + + // Even if something goes wrong with getting keys, and needs rotation returns a false positive, + // if there's no PendingDEK nothing happens. A snapshot is spuriously triggered, but it writes + // with the current DEK. + + // propose another value, so snapshotting doesn't fail + values[6], err = raftutils.ProposeValue(t, nodes[1], DefaultProposalTime, nodeIDs[6]) + require.NoError(t, err, "failed to propose value") + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:7], values[:7]) + + fakeTrue := true + nodes[1].KeyRotator.SetNeedsRotation(&fakeTrue) + nodes[1].KeyRotator.RotationNotify() <- struct{}{} + + require.NoError(t, raftutils.PollFunc(clockSource, func() error { + snapshots, err := storage.ListSnapshots(snapDir) + if err != nil { + return err + } + if len(snapshots) != 1 { + return fmt.Errorf("expected 1 snapshots, found %d on new node", len(snapshots)) + } + if snapshots[0] == currSnapshot { + return fmt.Errorf("new snapshot not done yet") + } + return nil + })) + + nodes[1].Server.Stop() + nodes[1].ShutdownRaft() + + // We can decrypt with key4 - no key5 + nodes[1].KeyRotator = raftutils.NewSimpleKeyRotator(raft.EncryptionKeys{ + CurrentDEK: []byte("key4"), + }) + nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], false) + raftutils.WaitForCluster(t, clockSource, nodes) + raftutils.CheckValuesOnNodes(t, clockSource, nodes, nodeIDs[:7], values[:7]) } diff --git a/manager/state/raft/testutils/testutils.go b/manager/state/raft/testutils/testutils.go index 4c92d1c3c0..2c34b44d23 100644 --- a/manager/state/raft/testutils/testutils.go +++ b/manager/state/raft/testutils/testutils.go @@ -5,6 +5,7 @@ import ( "net" "os" "reflect" + "sync" "testing" "time" @@ -35,6 +36,7 @@ type TestNode struct { Address string StateDir string cancel context.CancelFunc + KeyRotator *SimpleKeyRotator } // Leader is wrapper around real Leader method to suppress error. @@ -201,6 +203,79 @@ func RecycleWrappedListener(old *WrappedListener) *WrappedListener { } } +// SimpleKeyRotator does some DEK rotation +type SimpleKeyRotator struct { + mu sync.Mutex + rotateCh chan struct{} + updateFunc func() error + overrideNeedRotate *bool + raft.EncryptionKeys +} + +// GetKeys returns the current set of keys +func (s *SimpleKeyRotator) GetKeys() raft.EncryptionKeys { + s.mu.Lock() + defer s.mu.Unlock() + return s.EncryptionKeys +} + +// NeedsRotation returns whether we need to rotate +func (s *SimpleKeyRotator) NeedsRotation() bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.overrideNeedRotate != nil { + return *s.overrideNeedRotate + } + return s.EncryptionKeys.PendingDEK != nil +} + +// UpdateKeys updates the current encryption keys +func (s *SimpleKeyRotator) UpdateKeys(newKeys raft.EncryptionKeys) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.updateFunc != nil { + return s.updateFunc() + } + s.EncryptionKeys = newKeys + return nil +} + +// RotationNotify returns the rotation notification channel +func (s *SimpleKeyRotator) RotationNotify() chan struct{} { + return s.rotateCh +} + +// QueuePendingKey lets us rotate the key +func (s *SimpleKeyRotator) QueuePendingKey(key []byte) { + s.mu.Lock() + defer s.mu.Unlock() + s.EncryptionKeys.PendingDEK = key +} + +// SetUpdateFunc enables you to inject an error when updating keys +func (s *SimpleKeyRotator) SetUpdateFunc(updateFunc func() error) { + s.mu.Lock() + defer s.mu.Unlock() + s.updateFunc = updateFunc +} + +// SetNeedsRotation enables you to inject a value for NeedsRotation +func (s *SimpleKeyRotator) SetNeedsRotation(override *bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.overrideNeedRotate = override +} + +// NewSimpleKeyRotator returns a basic EncryptionKeyRotator +func NewSimpleKeyRotator(keys raft.EncryptionKeys) *SimpleKeyRotator { + return &SimpleKeyRotator{ + rotateCh: make(chan struct{}), + EncryptionKeys: keys, + } +} + +var _ raft.EncryptionKeyRotator = NewSimpleKeyRotator(raft.EncryptionKeys{}) + // NewNode creates a new raft node to use for tests func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, opts ...raft.NodeOptions) *TestNode { l, err := net.Listen("tcp", "127.0.0.1:0") @@ -218,6 +293,7 @@ func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, stateDir, err := ioutil.TempDir("", "test-raft") require.NoError(t, err, "can't create temporary state directory") + keyRotator := NewSimpleKeyRotator(raft.EncryptionKeys{CurrentDEK: []byte("current")}) newNodeOpts := raft.NodeOptions{ ID: securityConfig.ClientTLSCreds.NodeID(), Addr: l.Addr().String(), @@ -226,6 +302,7 @@ func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, ClockSource: clockSource, SendTimeout: 10 * time.Second, TLSCredentials: securityConfig.ClientTLSCreds, + KeyRotator: keyRotator, } if len(opts) > 1 { @@ -258,6 +335,7 @@ func NewNode(t *testing.T, clockSource *fakeclock.FakeClock, tc *cautils.TestCA, Address: newNodeOpts.Addr, StateDir: newNodeOpts.StateDir, Server: s, + KeyRotator: keyRotator, } } @@ -316,8 +394,8 @@ func NewJoinNode(t *testing.T, clockSource *fakeclock.FakeClock, join string, tc return n } -// RestartNode restarts a raft test node -func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNode, forceNewCluster bool) *TestNode { +// CopyNode returns a copy of a node +func CopyNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNode, forceNewCluster bool, kr *SimpleKeyRotator) (*TestNode, context.Context) { wrappedListener := RecycleWrappedListener(oldNode.Listener) securityConfig := oldNode.SecurityConfig serverOpts := []grpc.ServerOption{grpc.Creds(securityConfig.ServerTLSCreds)} @@ -325,6 +403,10 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo cfg := raft.DefaultNodeConfig() + if kr == nil { + kr = oldNode.KeyRotator + } + newNodeOpts := raft.NodeOptions{ ID: securityConfig.ClientTLSCreds.NodeID(), Addr: oldNode.Address, @@ -334,6 +416,7 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo ClockSource: clockSource, SendTimeout: 10 * time.Second, TLSCredentials: securityConfig.ClientTLSCreds, + KeyRotator: kr, } ctx, cancel := context.WithCancel(context.Background()) @@ -345,16 +428,11 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo go func() { // After stopping, we should receive an error from Serve - assert.Error(t, s.Serve(wrappedListener)) + require.Error(t, s.Serve(wrappedListener)) }() healthServer.SetServingStatus("Raft", api.HealthCheckResponse_SERVING) - err := n.JoinAndStart(ctx) - require.NoError(t, err, "can't join cluster") - - go n.Run(ctx) - return &TestNode{ Node: n, Listener: wrappedListener, @@ -363,7 +441,20 @@ func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNo StateDir: newNodeOpts.StateDir, cancel: cancel, Server: s, - } + KeyRotator: kr, + }, ctx +} + +// RestartNode restarts a raft test node +func RestartNode(t *testing.T, clockSource *fakeclock.FakeClock, oldNode *TestNode, forceNewCluster bool) *TestNode { + n, ctx := CopyNode(t, clockSource, oldNode, forceNewCluster, nil) + + err := n.Node.JoinAndStart(ctx) + require.NoError(t, err, "can't join cluster") + + go n.Node.Run(ctx) + + return n } // NewRaftCluster creates a new raft cluster with 3 nodes for testing diff --git a/node/node.go b/node/node.go index 1baa7216a6..2cc3b4bb25 100644 --- a/node/node.go +++ b/node/node.go @@ -496,7 +496,7 @@ func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, erro paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory)) var securityConfig *ca.SecurityConfig - krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, nil) + krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{}) if err := krw.Migrate(); err != nil { return nil, err } @@ -523,20 +523,19 @@ func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, erro } if securityConfig == nil { - switch { - case n.config.JoinAddr == "": + if n.config.JoinAddr == "" { // if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key n.unlockKey = nil if n.config.AutoLockManagers { n.unlockKey = encryption.GenerateSecretKey() } - krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, nil) + krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{}) rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA) if err != nil { return nil, err } log.G(ctx).Debug("generated CA key and certificate") - case err == ca.ErrNoLocalRootCA: + } else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes) if err != nil { return nil, err diff --git a/node/node_test.go b/node/node_test.go index c3caa279ef..ef74912ef5 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -9,9 +9,11 @@ import ( "path/filepath" "testing" + "github.com/docker/swarmkit/api" "github.com/docker/swarmkit/ca" cautils "github.com/docker/swarmkit/ca/testutils" "github.com/docker/swarmkit/identity" + "github.com/docker/swarmkit/manager/state/store" "github.com/pkg/errors" "github.com/stretchr/testify/require" ) @@ -179,6 +181,10 @@ func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) { _, err = node.loadSecurityConfig(context.Background()) require.NoError(t, err) + // the TLS key and cert were written to disk unencrypted + _, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read() + require.NoError(t, err) + // remove the TLS cert and key, and mark the root CA cert so that we will // know if it gets replaced require.NoError(t, os.Remove(paths.Node.Cert)) @@ -191,6 +197,25 @@ func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) { certBytes = pem.EncodeToMemory(pemBlock) require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, certBytes, 0644)) + // also make sure the new set gets downloaded and written to disk with a passphrase + // by updating the memory store with manager autolock on and an unlock key + require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error { + clusters, err := store.FindClusters(tx, store.All) + require.NoError(t, err) + require.Len(t, clusters, 1) + + newCluster := clusters[0].Copy() + newCluster.Spec.EncryptionConfig.AutoLockManagers = true + newCluster.UnlockKeys = []*api.EncryptionKey{{ + Subsystem: ca.ManagerRole, + Key: []byte("passphrase"), + }} + return store.UpdateCluster(tx, newCluster) + })) + + // Join with without any passphrase - this should be fine, because the TLS + // key is downloaded and then loaded just fine. However, it *is* written + // to disk encrypted. node, err = New(&Config{ StateDir: tempdir, JoinAddr: peer.Addr, @@ -204,4 +229,10 @@ func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) { readCertBytes, err := ioutil.ReadFile(paths.RootCA.Cert) require.NoError(t, err) require.Equal(t, certBytes, readCertBytes) + + // the TLS node cert and key were saved to disk encrypted, though + _, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read() + require.Error(t, err) + _, _, err = ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil).Read() + require.NoError(t, err) }