Skip to content

Commit 61bc2e4

Browse files
gagan-gauravgagan-gaurav
authored
feat: Added Notion Source Plugin (#13979)
 #### Summary Created Notion Source Plugin. This PR Closes #11943 <!--
1 parent 82bc09c commit 61bc2e4

34 files changed

Lines changed: 1515 additions & 2 deletions

‎.github/pr_labeler.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ mysql:
8686
- plugins/destination/mysql/**/*
8787
neo4j:
8888
- plugins/destination/neo4j/**/*
89+
notion:
90+
- plugins/source/notion/**/*
8991
okta:
9092
- plugins/source/okta/**/*
9193
oracle:

‎.github/workflows/source_notion.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Source Plugin Notion Workflow
2+
3+
concurrency:
4+
group: ${{ github.workflow }}-${{ github.ref }}
5+
cancel-in-progress: true
6+
7+
on:
8+
pull_request:
9+
paths:
10+
- "plugins/source/notion/**"
11+
- ".github/workflows/source_notion.yml"
12+
push:
13+
branches:
14+
- main
15+
paths:
16+
- "plugins/source/notion/**"
17+
- ".github/workflows/source_notion.yml"
18+
19+
jobs:
20+
plugins-source-notion:
21+
timeout-minutes: 30
22+
name: "plugins/source/notion"
23+
runs-on: large-ubuntu-monorepo
24+
defaults:
25+
run:
26+
working-directory: ./plugins/source/notion
27+
steps:
28+
- uses: actions/checkout@v3
29+
with:
30+
fetch-depth: 2
31+
- name: Set up Go 1.x
32+
uses: actions/setup-go@v3
33+
with:
34+
go-version-file: plugins/source/notion/go.mod
35+
cache: true
36+
cache-dependency-path: plugins/source/notion/go.sum
37+
- name: golangci-lint
38+
uses: golangci/golangci-lint-action@v3
39+
with:
40+
version: v1.52.2
41+
working-directory: plugins/source/notion
42+
args: "--config ../../.golangci.yml"
43+
skip-pkg-cache: true
44+
skip-build-cache: true
45+
- name: Setup CloudQuery
46+
if: github.event_name == 'pull_request'
47+
uses: cloudquery/setup-cloudquery@v3
48+
with:
49+
version: 'v3.13.0'
50+
- name: gen
51+
if: github.event_name == 'pull_request'
52+
run: make gen
53+
- name: Fail if generation updated files
54+
if: github.event_name == 'pull_request'
55+
run: test "$(git status -s | wc -l)" -eq 0 || (git status -s; exit 1)
56+
- name: Build
57+
run: go build .
58+
- name: Test
59+
run: make test
60+
validate-release:
61+
timeout-minutes: 30
62+
runs-on: large-ubuntu-monorepo
63+
env:
64+
CGO_ENABLED: 0
65+
steps:
66+
- name: Checkout
67+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
68+
uses: actions/checkout@v3
69+
- uses: actions/cache@v3
70+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
71+
with:
72+
path: |
73+
~/.cache/go-build
74+
~/go/pkg/mod
75+
key: ${{ runner.os }}-go-1.20.5-release-cache-${{ hashFiles('plugins/source/notion/go.sum') }}
76+
restore-keys: |
77+
${{ runner.os }}-go-1.20.5-release-cache-plugins-source-notion
78+
- name: Set up Go
79+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
80+
uses: actions/setup-go@v3
81+
with:
82+
go-version-file: plugins/source/notion/go.mod
83+
- name: Install GoReleaser
84+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
85+
uses: goreleaser/goreleaser-action@v3
86+
with:
87+
distribution: goreleaser-pro
88+
version: latest
89+
install-only: true
90+
- name: Run GoReleaser Dry-Run
91+
if: startsWith(github.head_ref, 'release-please--branches--main--components') || github.event_name == 'push'
92+
run: goreleaser release --snapshot --clean --skip-validate --skip-publish --skip-sign -f ./plugins/source/notion/.goreleaser.yaml
93+
env:
94+
GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}

‎plugins/source/notion/.gitignore‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Binaries for programs and plugins
2+
*.exe
3+
*.exe~
4+
*.dll
5+
*.so
6+
*.dylib
7+
8+
# Test binary, built with `go test -c`
9+
*.test
10+
11+
# Output of the go coverage tool, specifically when used with LiteIDE
12+
*.out
13+
14+
# Dependency directories (remove the comment below to include it)
15+
vendor/
16+
17+
18+
.DS_Store
19+
20+
notion
21+
22+
dist
23+
*.log

‎plugins/source/notion/.goreleaser.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
variables:
2+
component: source/notion
3+
binary: notion
4+
5+
project_name: plugins/source/notion
6+
7+
monorepo:
8+
tag_prefix: plugins-source-notion-
9+
dir: plugins/source/notion
10+
11+
includes:
12+
- from_file:
13+
# Relative to the directory Go Releaser is run from (which is the root of the repository)
14+
path: ./plugins/.goreleaser.yaml

‎plugins/source/notion/Makefile‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
.PHONY: test
2+
test:
3+
go test -timeout 3m ./...
4+
5+
.PHONY: build
6+
build:
7+
go build -o notion
8+
9+
.PHONY: lint
10+
lint:
11+
@golangci-lint run --timeout 10m
12+
13+
PHONY: gen-docs
14+
gen-docs: build
15+
@command -v cloudquery >/dev/null 2>&1 || { \
16+
echo "Error: 'cloudquery' command not found. Please install it before running gen-docs."; \
17+
echo "You can install it by following the instructions at: https://www.cloudquery.io/docs/quickstart"; \
18+
exit 1; \
19+
}
20+
rm -rf ../../../website/tables/notion
21+
cloudquery tables --format markdown --output-dir ../../../website/tables test/config.yml
22+
sed 's_(\(.*\))_(../../../../../website/tables/notion/\1)_' ../../../website/tables/notion/README.md > ./docs/tables/README.md
23+
sed -i.bak -e 's_(\(.*\).md)_(tables/\1)_' ../../../website/tables/notion/README.md
24+
mkdir --parents ../../../website/pages/docs/plugins/sources/notion/
25+
mv ../../../website/tables/notion/README.md ../../../website/pages/docs/plugins/sources/notion/tables.md
26+
sed -i.bak -e 's_(\(.*\).md)_(\1)_' ../../../website/tables/notion/*.md
27+
rm -rf ../../../website/tables/notion/*.bak
28+
29+
# All gen targets
30+
.PHONY: gen
31+
gen: gen-docs

‎plugins/source/notion/README.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# CloudQuery notion Source Plugin
2+
3+
A notion source plugin for CloudQuery that loads data from notion to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
4+
5+
## Links
6+
7+
- [CloudQuery Quickstart Guide](https://www.cloudquery.io/docs/quickstart)
8+
- [Supported Tables](docs/tables/README.md)
9+
10+
## Authentication
11+
12+
In Order for CloudQuery to sync resources from your Notion setup, you will need to create a notion integration key and export the Token in NOTION_SECRET_KEY environment variable.
13+
How to create the notion integration key? [see here](https://developers.notion.com/docs/create-a-notion-integration#create-your-integration-in-notion). Make sure to give proper **Content Capabilities** and **User Capabilities** from capabilities settings. Also give your integration page permissions [see here](https://developers.notion.com/docs/create-a-notion-integration#give-your-integration-page-permissions). Only pages and databases with permission will able to sync.
14+
15+
```bash
16+
export NOTION_SECRET_KEY=<your_notion_integration_key>
17+
```
18+
19+
## Configuration
20+
21+
The following source configuration file will sync to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination.
22+
23+
```yaml
24+
kind: source
25+
spec:
26+
name: "notion"
27+
path: "cloudquery/notion"
28+
version: "${VERSION}"
29+
destinations:
30+
- "postgresql"
31+
spec:
32+
bearer_token: "${NOTION_SECRET_KEY}"
33+
```
34+
35+

‎plugins/source/notion/client/client.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package client
2+
3+
import (
4+
"context"
5+
6+
"github.com/rs/zerolog"
7+
)
8+
9+
type Client struct {
10+
logger zerolog.Logger
11+
Spec Spec
12+
Notion *NotionClient
13+
}
14+
15+
func (Client) ID() string {
16+
return "notion"
17+
}
18+
19+
func (c *Client) Logger() *zerolog.Logger {
20+
return &c.logger
21+
}
22+
23+
func New(ctx context.Context, logger zerolog.Logger, s *Spec) (Client, error) {
24+
s.SetDefaults() // Sets the default value of Notion Version.
25+
bearerToken := s.BearerToken
26+
notionVersion := s.NotionVersion
27+
c, err := NewNotionClient(bearerToken, notionVersion)
28+
if err != nil {
29+
return Client{}, err
30+
}
31+
32+
return Client{
33+
logger: logger,
34+
Spec: *s,
35+
Notion: c,
36+
}, nil
37+
}

0 commit comments

Comments
 (0)