Skip to content

Commit 8a98478

Browse files
matheuscscpmatheuscscp
committed
ssa: introduce custom apply stage
Signed-off-by: Matheus Pimenta <matheuscscp@gmail.com>
1 parent 2818265 commit 8a98478

5 files changed

Lines changed: 307 additions & 1 deletion

File tree

‎ssa/manager_apply.go‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"k8s.io/apimachinery/pkg/api/errors"
2929
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3030
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
31+
"k8s.io/apimachinery/pkg/runtime/schema"
3132
"k8s.io/apimachinery/pkg/types"
3233
"k8s.io/apimachinery/pkg/util/wait"
3334
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -63,6 +64,10 @@ type ApplyOptions struct {
6364

6465
// Cleanup defines which in-cluster metadata entries are to be removed before applying objects.
6566
Cleanup ApplyCleanupOptions `json:"cleanup"`
67+
68+
// CustomStageKinds defines a set of Kubernetes resource types that should be applied
69+
// in a separate stage after CRDs and before namespaced objects.
70+
CustomStageKinds map[schema.GroupKind]struct{} `json:"customStageKinds,omitempty"`
6671
}
6772

6873
// ApplyCleanupOptions defines which metadata entries are to be removed before applying objects.
@@ -154,7 +159,6 @@ func (m *ResourceManager) ApplyAll(ctx context.Context, objects []*unstructured.
154159
g, ctx := errgroup.WithContext(ctx)
155160
g.SetLimit(m.concurrency)
156161
for i, object := range objects {
157-
i, object := i, object
158162

159163
g.Go(func() error {
160164
utils.RemoveCABundleFromCRD(object)
@@ -259,6 +263,9 @@ func (m *ResourceManager) ApplyAllStaged(ctx context.Context, objects []*unstruc
259263
// Contains only Class definitions.
260264
classStage []*unstructured.Unstructured
261265

266+
// Contains the custom kinds.
267+
customStage []*unstructured.Unstructured
268+
262269
// Contains all objects except for cluster definitions and class definitions.
263270
resStage []*unstructured.Unstructured
264271
)
@@ -269,6 +276,8 @@ func (m *ResourceManager) ApplyAllStaged(ctx context.Context, objects []*unstruc
269276
defStage = append(defStage, o)
270277
case utils.IsClassDefinition(o):
271278
classStage = append(classStage, o)
279+
case utils.IsCustomStage(o, opts.CustomStageKinds):
280+
customStage = append(customStage, o)
272281
default:
273282
resStage = append(resStage, o)
274283
}
@@ -300,6 +309,15 @@ func (m *ResourceManager) ApplyAllStaged(ctx context.Context, objects []*unstruc
300309
}
301310
}
302311

312+
// Apply custom staged objects next.
313+
if len(customStage) > 0 {
314+
cs, err := m.ApplyAll(ctx, customStage, opts)
315+
if err != nil {
316+
return changeSet, err
317+
}
318+
changeSet.Append(cs.Entries)
319+
}
320+
303321
// Finally, apply all the other resources.
304322
cs, err := m.ApplyAll(ctx, resStage, opts)
305323
if err != nil {

‎ssa/manager_apply_test.go‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package ssa
2020
import (
2121
"context"
2222
"encoding/base64"
23+
"errors"
2324
"fmt"
2425
"sort"
2526
"strings"
@@ -29,11 +30,14 @@ import (
2930
"github.com/google/go-cmp/cmp"
3031
corev1 "k8s.io/api/core/v1"
3132
rbacv1 "k8s.io/api/rbac/v1"
33+
apierrors "k8s.io/apimachinery/pkg/api/errors"
3234
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3335
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
36+
"k8s.io/apimachinery/pkg/runtime/schema"
3437
"k8s.io/client-go/rest"
3538
"sigs.k8s.io/controller-runtime/pkg/client"
3639

40+
ssaerrors "github.com/fluxcd/pkg/ssa/errors"
3741
"github.com/fluxcd/pkg/ssa/normalize"
3842
"github.com/fluxcd/pkg/ssa/utils"
3943
)
@@ -1473,3 +1477,186 @@ func TestResourceManager_ApplyAllStaged_CRDWebhookCABundle(t *testing.T) {
14731477
}
14741478
})
14751479
}
1480+
1481+
func TestApplyAllStaged_AppliesRoleAndRoleBinding(t *testing.T) {
1482+
timeout := 10 * time.Second
1483+
ctx, cancel := context.WithTimeout(context.Background(), timeout)
1484+
defer cancel()
1485+
1486+
id := generateName("custom-stage")
1487+
1488+
// Create a non-cluster-admin client to ensure dry-run checks are not bypassed.
1489+
customStageNS := &corev1.Namespace{
1490+
ObjectMeta: metav1.ObjectMeta{
1491+
Name: id,
1492+
},
1493+
}
1494+
if err := manager.client.Create(ctx, customStageNS); err != nil {
1495+
t.Fatal(err)
1496+
}
1497+
customStageSA := &corev1.ServiceAccount{
1498+
ObjectMeta: metav1.ObjectMeta{
1499+
Name: id,
1500+
Namespace: id,
1501+
},
1502+
}
1503+
if err := manager.client.Create(ctx, customStageSA); err != nil {
1504+
t.Fatal(err)
1505+
}
1506+
customStageCR := &rbacv1.ClusterRole{
1507+
ObjectMeta: metav1.ObjectMeta{
1508+
Name: id,
1509+
},
1510+
Rules: []rbacv1.PolicyRule{
1511+
{
1512+
APIGroups: []string{"rbac.authorization.k8s.io"},
1513+
Resources: []string{"roles", "rolebindings"},
1514+
Verbs: []string{"create", "update", "delete", "get", "list", "watch", "patch"},
1515+
},
1516+
// Grant the same permissions that the test Role will grant,
1517+
// so RBAC escalation prevention allows creating the Role and
1518+
// RoleBinding.
1519+
{
1520+
APIGroups: []string{""},
1521+
Resources: []string{"configmaps"},
1522+
Verbs: []string{"get", "list"},
1523+
},
1524+
},
1525+
}
1526+
if err := manager.client.Create(ctx, customStageCR); err != nil {
1527+
t.Fatal(err)
1528+
}
1529+
customStageCRB := &rbacv1.ClusterRoleBinding{
1530+
ObjectMeta: metav1.ObjectMeta{
1531+
Name: id,
1532+
},
1533+
RoleRef: rbacv1.RoleRef{
1534+
APIGroup: rbacv1.GroupName,
1535+
Kind: "ClusterRole",
1536+
Name: customStageCR.Name,
1537+
},
1538+
Subjects: []rbacv1.Subject{{
1539+
Kind: "ServiceAccount",
1540+
Name: customStageSA.Name,
1541+
Namespace: customStageSA.Namespace,
1542+
}},
1543+
}
1544+
if err := manager.client.Create(ctx, customStageCRB); err != nil {
1545+
t.Fatal(err)
1546+
}
1547+
1548+
// Create a manager with the non-cluster-admin client
1549+
customStageManager := *manager
1550+
customStageCfg := rest.CopyConfig(cfg)
1551+
customStageCfg.Impersonate = rest.ImpersonationConfig{
1552+
UserName: fmt.Sprintf("system:serviceaccount:%s:%s", customStageSA.Namespace, customStageSA.Name),
1553+
}
1554+
customStageClient, err := client.New(customStageCfg, client.Options{
1555+
Mapper: restMapper,
1556+
})
1557+
if err != nil {
1558+
t.Fatal(err)
1559+
}
1560+
customStageManager.client = customStageClient
1561+
1562+
// Create a Role and RoleBinding that references the Role.
1563+
role := &unstructured.Unstructured{
1564+
Object: map[string]any{
1565+
"apiVersion": "rbac.authorization.k8s.io/v1",
1566+
"kind": "Role",
1567+
"metadata": map[string]any{
1568+
"name": "role",
1569+
"namespace": id,
1570+
},
1571+
"rules": []any{
1572+
map[string]any{
1573+
"apiGroups": []any{""},
1574+
"resources": []any{"configmaps"},
1575+
"verbs": []any{"get", "list"},
1576+
},
1577+
},
1578+
},
1579+
}
1580+
1581+
roleBinding := &unstructured.Unstructured{
1582+
Object: map[string]any{
1583+
"apiVersion": "rbac.authorization.k8s.io/v1",
1584+
"kind": "RoleBinding",
1585+
"metadata": map[string]any{
1586+
"name": "role-binding",
1587+
"namespace": id,
1588+
},
1589+
"roleRef": map[string]any{
1590+
"apiGroup": "rbac.authorization.k8s.io",
1591+
"kind": "Role",
1592+
"name": "role",
1593+
},
1594+
"subjects": []any{
1595+
map[string]any{
1596+
"kind": "ServiceAccount",
1597+
"name": "default",
1598+
"namespace": id,
1599+
},
1600+
},
1601+
},
1602+
}
1603+
1604+
objects := []*unstructured.Unstructured{roleBinding, role}
1605+
1606+
t.Run("does not apply Role and RoleBinding together without custom stage", func(t *testing.T) {
1607+
opts := DefaultApplyOptions()
1608+
1609+
_, err := customStageManager.ApplyAllStaged(ctx, objects, opts)
1610+
if err == nil {
1611+
t.Fatal("Expected error when applying RoleBinding before Role, got none")
1612+
}
1613+
1614+
// Assert the error is a DryRunErr
1615+
var dryRunErr *ssaerrors.DryRunErr
1616+
if !errors.As(err, &dryRunErr) {
1617+
t.Fatalf("Expected error to be *errors.DryRunErr, got %T", err)
1618+
}
1619+
1620+
// Assert the underlying error is NotFound
1621+
if !apierrors.IsNotFound(dryRunErr.Unwrap()) {
1622+
t.Errorf("Expected underlying error to be NotFound, got: %v", dryRunErr.Unwrap())
1623+
}
1624+
1625+
// Assert the NotFound is for the Role that the RoleBinding references
1626+
var statusErr *apierrors.StatusError
1627+
if !errors.As(dryRunErr.Unwrap(), &statusErr) {
1628+
t.Fatalf("Expected underlying error to be *apierrors.StatusError, got %T", dryRunErr.Unwrap())
1629+
}
1630+
if statusErr.ErrStatus.Details == nil || statusErr.ErrStatus.Details.Name != "role" {
1631+
t.Errorf("Expected NotFound to be for the Role named 'role', got: %+v", statusErr.ErrStatus.Details)
1632+
}
1633+
1634+
// Assert the involved object is the RoleBinding
1635+
if dryRunErr.InvolvedObject().GetKind() != "RoleBinding" {
1636+
t.Errorf("Expected involved object to be RoleBinding, got %s", dryRunErr.InvolvedObject().GetKind())
1637+
}
1638+
})
1639+
1640+
t.Run("applies Role and RoleBinding together with Role in custom stage", func(t *testing.T) {
1641+
opts := DefaultApplyOptions()
1642+
opts.CustomStageKinds = map[schema.GroupKind]struct{}{
1643+
{Group: "rbac.authorization.k8s.io", Kind: "Role"}: {},
1644+
}
1645+
1646+
changeSet, err := customStageManager.ApplyAllStaged(ctx, objects, opts)
1647+
if err != nil {
1648+
t.Fatal(err)
1649+
}
1650+
1651+
// Verify both objects were created
1652+
if len(changeSet.Entries) != 2 {
1653+
t.Errorf("Expected 2 entries, got %d", len(changeSet.Entries))
1654+
}
1655+
1656+
for _, entry := range changeSet.Entries {
1657+
if diff := cmp.Diff(entry.Action, CreatedAction); diff != "" {
1658+
t.Errorf("Mismatch from expected value (-want +got):\n%s", diff)
1659+
}
1660+
}
1661+
})
1662+
}

‎ssa/utils/is.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"strings"
2121

2222
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
23+
"k8s.io/apimachinery/pkg/runtime/schema"
2324
)
2425

2526
// IsClusterDefinition checks if the given object is a Kubernetes
@@ -54,6 +55,12 @@ func IsClassDefinition(object *unstructured.Unstructured) bool {
5455
return strings.HasSuffix(object.GetKind(), "Class")
5556
}
5657

58+
// IsCustomStage checks if the given object matches any of the provided custom stage kinds.
59+
func IsCustomStage(object *unstructured.Unstructured, customStageKinds map[schema.GroupKind]struct{}) bool {
60+
_, exists := customStageKinds[object.GroupVersionKind().GroupKind()]
61+
return exists
62+
}
63+
5764
// IsClusterRole checks if the given object is a Kubernetes ClusterRole definition.
5865
func IsClusterRole(object *unstructured.Unstructured) bool {
5966
return strings.ToLower(object.GetKind()) == "clusterrole" &&

‎ssa/utils/meta.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,11 @@ limitations under the License.
1717
package utils
1818

1919
import (
20+
"fmt"
2021
"strings"
2122

2223
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
24+
"k8s.io/apimachinery/pkg/runtime/schema"
2325
)
2426

2527
// SetCommonMetadata adds the specified labels and annotations to all objects.
@@ -67,3 +69,24 @@ func AnyInMetadata(object *unstructured.Unstructured, metadata map[string]string
6769
}
6870
return false
6971
}
72+
73+
// ParseGroupKindSet converts a string of the form "group1/kind1,group2/kind2"
74+
// into a set of GroupKind.
75+
func ParseGroupKindSet(s string) (map[schema.GroupKind]struct{}, error) {
76+
set := make(map[schema.GroupKind]struct{})
77+
if s == "" {
78+
return set, nil
79+
}
80+
for item := range strings.SplitSeq(s, ",") {
81+
parts := strings.SplitN(strings.TrimSpace(item), "/", 2)
82+
switch len(parts) {
83+
case 1:
84+
set[schema.GroupKind{Group: "", Kind: parts[0]}] = struct{}{}
85+
case 2:
86+
set[schema.GroupKind{Group: parts[0], Kind: parts[1]}] = struct{}{}
87+
default:
88+
return nil, fmt.Errorf("invalid group/kind format: %s", item)
89+
}
90+
}
91+
return set, nil
92+
}

0 commit comments

Comments
 (0)