feat(gcp)!: Make Listing Enabled Services failures not block the sync (#10699)

bbernays · web-flow · commit 5b49481f53f8 · 2023-05-15T15:01:26.000Z
#### Summary
We currently require a successful `ListServices` for all projects in the org. If a user cannot make the call for a single project then the entire sync will fail. This PR changes the listing to be a best effort, which means that if CQ is not able to make the `ListServices` call it will just assume that all services are enabled
diff --git a/plugins/source/gcp/client/client.go b/plugins/source/gcp/client/client.go
@@ -276,7 +276,7 @@ func New(ctx context.Context, logger zerolog.Logger, s specs.Source, opts source
 		c.ProjectId = projects[0]
 	}
 	if gcpSpec.EnabledServicesOnly {
-		if err := c.configureEnabledServices(ctx, s.Concurrency); err != nil {
+		if err := c.configureEnabledServices(ctx, int(s.Concurrency)); err != nil {
 			return nil, err
 		}
 	}
@@ -482,24 +482,31 @@ func setUnion(a []string, b []string) []string {
 	return union
 }
 
-func (c *Client) configureEnabledServices(ctx context.Context, concurrency uint64) error {
+func (c *Client) configureEnabledServices(ctx context.Context, concurrency int) error {
 	var esLock sync.Mutex
 	g, ctx := errgroup.WithContext(ctx)
-	g.SetLimit(int(concurrency))
-
+	g.SetLimit(concurrency)
 	for _, p := range c.projects {
 		project := p
 		g.Go(func() error {
 			cl := c.withProject(project)
 			svc, err := cl.fetchEnabledServices(ctx)
+			if err != nil {
+				switch status.Code(err) {
+				case codes.ResourceExhausted:
+					c.logger.Warn().Err(err).Msgf("failed to list enabled services because of rate limiting. Sync will continue without filtering out disabled services for this project: %s. Consider setting larger values for `backoff_retries` and `backoff_delay`", project)
+				case codes.PermissionDenied:
+					c.logger.Warn().Err(err).Msgf("failed to list enabled services because of insufficient permissions. Sync will continue without filtering out disabled services for this project: %s", project)
+				default:
+					c.logger.Err(err).Msg("failed to list enabled services")
+					return err
+				}
+				return nil
+			}
+			// Only update the enabled services if we were able to list all services successfully
 			esLock.Lock()
 			c.EnabledServices[project] = svc
 			esLock.Unlock()
-			if status.Code(err) == codes.ResourceExhausted {
-				c.logger.Err(err).Msg("failed to list enabled services because of rate limiting. Consider setting larger values for `backoff_retries` and `backoff_delay`")
-			} else {
-				c.logger.Err(err).Msg("failed to list enabled services")
-			}
 			return err
 		})
 	}
diff --git a/plugins/source/gcp/client/multiplex.go b/plugins/source/gcp/client/multiplex.go
@@ -24,10 +24,18 @@ func ProjectMultiplexEnabledServices(enabledService string) func(schema.ClientMe
 		// preallocate all clients just in case
 		l := make([]schema.ClientMeta, 0, len(cl.projects))
 		for _, projectId := range cl.projects {
-			// This map can only be empty if user has not opted into `EnabledServicesOnly` via the spec
+			// When EnabledServicesOnly is empty, we can assume that all services should be synced
 			if len(cl.EnabledServices) == 0 {
 				l = append(l, cl.withProject(projectId))
-			} else if cl.EnabledServices[projectId] != nil && cl.EnabledServices[projectId][enabledService] != nil {
+				continue
+			}
+			// When EnabledServices[projectId] is nil then we can assume that all services should be synced as the Listing of Enabled Services must have failed
+			if cl.EnabledServices[projectId] == nil {
+				l = append(l, cl.withProject(projectId))
+				continue
+			}
+			// When `projectId` key is set then we can assume listing was completed successfully and only if the enabledService is set should we sync
+			if cl.EnabledServices[projectId] != nil && cl.EnabledServices[projectId][enabledService] != nil {
 				l = append(l, cl.withProject(projectId))
 			}
 		}
