feat(gitlab): Add Branches (#6135)

bbernays · web-flow · commit 31185ddcafbe · 2022-12-30T14:28:13.000Z
#### Summary

&lt;!--
Explain what problem this PR addresses
--&gt;

&lt;!--
diff --git a/plugins/source/gitlab/client/client.go b/plugins/source/gitlab/client/client.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cloudquery/cloudquery/plugins/source/gitlab/client/services"
 	"github.com/cloudquery/plugin-sdk/schema"
 	"github.com/cloudquery/plugin-sdk/specs"
 	"github.com/rs/zerolog"
@@ -15,17 +14,10 @@ type Client struct {
 	// This is a client that you need to create and initialize in Configure
 	// It will be passed for each resource fetcher.
 	logger  zerolog.Logger
-	Gitlab  Services
+	Gitlab  *gitlab.Client
 	spec    specs.Source
 	BaseURL string
 }
-type Services struct {
-	Users    services.UsersClient
-	Groups   services.GroupsClient
-	Projects services.ProjectsClient
-	Settings services.SettingsClient
-	Releases services.ReleasesClient
-}
 
 func (c *Client) Logger() *zerolog.Logger {
 	return &c.logger
@@ -56,18 +48,7 @@ func Configure(ctx context.Context, logger zerolog.Logger, s specs.Source) (sche
 
 	return &Client{
 		logger: logger,
-		Gitlab: NewServices(c),
+		Gitlab: c,
 		spec:   s,
 	}, nil
 }
-
-// Take gitlab.Client as an argument and return an initialized Services struct
-func NewServices(c *gitlab.Client) Services {
-	return Services{
-		Users:    c.Users,
-		Groups:   c.Groups,
-		Projects: c.Projects,
-		Settings: c.Settings,
-		Releases: c.Releases,
-	}
-}
diff --git a/plugins/source/gitlab/client/testing.go b/plugins/source/gitlab/client/testing.go
@@ -48,7 +48,7 @@ func GitlabMockTestHelper(t *testing.T, table *schema.Table, createService func(
 		}
 		c := &Client{
 			logger:  l,
-			Gitlab:  NewServices(client),
+			Gitlab:  client,
 			BaseURL: ts.URL,
 		}
 
diff --git a/plugins/source/gitlab/codegen/recipes/projects.go b/plugins/source/gitlab/codegen/recipes/projects.go
@@ -13,7 +13,20 @@ func Projects() []*Resource {
 			SubService: "projects",
 			PKColumns:  []string{"base_url", "id"},
 			Struct:     &gitlab.Project{},
-			Relations:  []string{"ProjectsReleases()"},
+			Relations:  []string{"ProjectsReleases()", "ProjectBranches()"},
+		},
+		{
+			Service:    "projects",
+			SubService: "project_branches",
+			Struct:     &gitlab.Branch{},
+			PKColumns:  []string{"project_id", "name"},
+			ExtraColumns: []codegen.ColumnDefinition{
+				{
+					Name:     "project_id",
+					Type:     schema.TypeInt,
+					Resolver: `resolveProjectID`,
+				},
+			},
 		},
 		{
 			Service:    "projects",
diff --git a/plugins/source/gitlab/docs/tables/README.md b/plugins/source/gitlab/docs/tables/README.md
@@ -6,5 +6,6 @@
   - [gitlab_group_members](gitlab_group_members.md)
 - [gitlab_projects](gitlab_projects.md)
   - [gitlab_projects_releases](gitlab_projects_releases.md)
+  - [gitlab_project_branches](gitlab_project_branches.md)
 - [gitlab_settings](gitlab_settings.md)
 - [gitlab_users](gitlab_users.md)
diff --git a/plugins/source/gitlab/docs/tables/gitlab_project_branches.md b/plugins/source/gitlab/docs/tables/gitlab_project_branches.md
@@ -0,0 +1,27 @@
+# Table: gitlab_project_branches
+
+The composite primary key for this table is (**project_id**, **name**).
+
+## Relations
+
+This table depends on [gitlab_projects](gitlab_projects.md).
+
+## Columns
+
+| Name          | Type          |
+| ------------- | ------------- |
+|_cq_source_name|String|
+|_cq_sync_time|Timestamp|
+|_cq_id|UUID|
+|_cq_parent_id|UUID|
+|base_url|String|
+|project_id (PK)|Int|
+|commit|JSON|
+|name (PK)|String|
+|protected|Bool|
+|merged|Bool|
+|default|Bool|
+|can_push|Bool|
+|developers_can_push|Bool|
+|developers_can_merge|Bool|
+|web_url|String|
diff --git a/plugins/source/gitlab/docs/tables/gitlab_projects.md b/plugins/source/gitlab/docs/tables/gitlab_projects.md
@@ -6,6 +6,7 @@ The composite primary key for this table is (**base_url**, **id**).
 
 The following tables depend on gitlab_projects:
   - [gitlab_projects_releases](gitlab_projects_releases.md)
+  - [gitlab_project_branches](gitlab_project_branches.md)
 
 ## Columns
 
diff --git a/plugins/source/gitlab/resources/services/projects/project_branches.go b/plugins/source/gitlab/resources/services/projects/project_branches.go
diff --git a/plugins/source/gitlab/resources/services/projects/projects.go b/plugins/source/gitlab/resources/services/projects/projects.go
diff --git a/plugins/source/gitlab/resources/services/projects/projects_branches_fetch.go b/plugins/source/gitlab/resources/services/projects/projects_branches_fetch.go
@@ -0,0 +1,37 @@
+package projects
+
+import (
+	"context"
+
+	"github.com/cloudquery/cloudquery/plugins/source/gitlab/client"
+	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/xanzy/go-gitlab"
+)
+
+func fetchProjectBranches(ctx context.Context, meta schema.ClientMeta, parent *schema.Resource, res chan<- any) error {
+	c := meta.(*client.Client)
+	project := parent.Item.(*gitlab.Project)
+	opt := &gitlab.ListBranchesOptions{
+		ListOptions: gitlab.ListOptions{
+			PerPage: 1000,
+		},
+	}
+
+	for {
+		// Get the first page with projects.
+		branches, resp, err := c.Gitlab.Branches.ListBranches(project.ID, opt, gitlab.WithContext(ctx))
+		if err != nil {
+			return err
+		}
+		res <- branches
+		// Exit the loop when we've seen all pages.
+		if resp.NextPage == 0 {
+			break
+		}
+
+		// Update the page number to get the next page.
+		opt.Page = resp.NextPage
+	}
+
+	return nil
+}
diff --git a/plugins/source/gitlab/resources/services/projects/projects_mock_test.go b/plugins/source/gitlab/resources/services/projects/projects_mock_test.go
@@ -53,6 +53,20 @@ func buildProjects(mux *httprouter.Router) error {
 	mux.GET("/api/v4/projects/:projectId/releases", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
 		fmt.Fprint(w, string(releaseResp))
 	})
+
+	var branch *gitlab.Branch
+	if err := faker.FakeObject(&branch); err != nil {
+		return err
+	}
+
+	branchResp, err := json.Marshal([]*gitlab.Branch{branch})
+	if err != nil {
+		return err
+	}
+
+	mux.GET("/api/v4/projects/:projectId/repository/branches", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprint(w, string(branchResp))
+	})
 	return nil
 }
 
diff --git a/website/pages/docs/plugins/sources/gitlab/tables.md b/website/pages/docs/plugins/sources/gitlab/tables.md
@@ -6,5 +6,6 @@
   - [gitlab_group_members](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gitlab/docs/tables/gitlab_group_members.md)
 - [gitlab_projects](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gitlab/docs/tables/gitlab_projects.md)
   - [gitlab_projects_releases](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gitlab/docs/tables/gitlab_projects_releases.md)
+  - [gitlab_project_branches](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gitlab/docs/tables/gitlab_project_branches.md)
 - [gitlab_settings](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gitlab/docs/tables/gitlab_settings.md)
 - [gitlab_users](https://github.com/cloudquery/cloudquery/blob/main/plugins/source/gitlab/docs/tables/gitlab_users.md)

Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ func GitlabMockTestHelper(t testing.T, table schema.Table, createService func(`
`48`	`48`	`}`
`49`	`49`	`c := &Client{`
`50`	`50`	`logger: l,`
`51`		`- Gitlab: NewServices(client),`
	`51`	`+ Gitlab: client,`
`52`	`52`	`BaseURL: ts.URL,`
`53`	`53`	`}`
`54`	`54`