Skip to content

Commit 02506c2

Browse files
zeeshanakram3zeeshanakram3
committed
add opentelemetry tracing integration with graphql-server and auth-server
1 parent e34ebbb commit 02506c2

14 files changed

Lines changed: 22930 additions & 16527 deletions

File tree

‎.env‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,3 +72,11 @@ EMAIL_NOTIFICATION_DELIVERY_MAX_ATTEMPTS=5
7272
APP_ASSET_STORAGE=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/logos/gleev
7373
APP_NAME_ALT=Gleev.xyz
7474
NOTIFICATION_ASSET_ROOT=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/icons
75+
76+
# =====================================================================================
77+
# Telemetry
78+
# =====================================================================================
79+
80+
# yes/no
81+
TELEMETRY_ENABLED=no
82+
TELEMETRY_ENDPOINT=http://apm-server:8200

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,4 @@ CMD ["npm", "run", "processor-start"]
4545

4646

4747
FROM squid AS query-node
48-
CMD ["npm", "run", "query-node-start"]
48+
CMD ["npm", "run", "graphql-server-start"]

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ build-docker:
1212
@docker build . -t joystream/orion
1313

1414
serve:
15-
@npx squid-graphql-server --subscriptions
15+
@npm run graphql-server-start
1616

1717
serve-auth-api:
1818
@npm run auth-server-start

‎docker-compose.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ services:
4747
- docker.env
4848
environment:
4949
- SQD_TRACE=authentication
50+
- OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
5051
depends_on:
5152
- orion_db
5253
volumes:
@@ -68,6 +69,7 @@ services:
6869
- docker.env
6970
environment:
7071
- SQD_TRACE=authentication
72+
- OTEL_EXPORTER_OTLP_ENDPOINT=${TELEMETRY_ENDPOINT}
7173
depends_on:
7274
- orion_db
7375
volumes:

‎docs/developer-guide/commands.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
- `npm run checks` - runs basic checks (linting, formatting, type checking), see: [Code style](./code-style.md)
2929
- `npm run db:migrate` - runs database migrations, same as `make migrate`
3030
- `npm run processor-start` - similar to `make process`, but doesn't run migrations
31-
- `npm run query-node-start` - similar to `make serve`
31+
- `npm run graphql-server-start` - similar to `make serve`
3232
- `npm run auth-server-start` - same as `make serve-auth-api`
3333
- `npm run tests:auth-api` - runs unit tests for the `auth-api` service (see: [Authentication API](./tutorials/authentication-api.md)).
3434
- `npm run offchain-state:export` - performs the offchain state export, see: [Preserving offchain state](./tutorials/preserving-offchain-state.md)

‎docs/operator-guide/examples/docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ services:
4848
environment:
4949
- DB_HOST=orion-db
5050
- GQL_PORT=${GRAPHQL_API_PORT}
51-
command: ['npm', 'run', 'query-node-start']
51+
command: ['npm', 'run', 'graphql-server-start']
5252
ports:
5353
- '127.0.0.1:${GRAPHQL_API_PORT}:${GRAPHQL_API_PORT}'
5454
# Orion Authentication API

‎entrypoints/auth-server.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
# docker entrypoint fot auth api, to allow running with telemetry
6+
if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
7+
node --require ./opentelemetry/index.js ./lib/auth-server/index.js $*
8+
else
9+
node ./lib/auth-server/index.js $*
10+
fi

‎entrypoints/graphql-server.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
# docker entrypoint fot graphql-server, to allow running with telemetry
6+
if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
7+
node --require ./opentelemetry/index.js ./node_modules/@subsquid/graphql-server/bin/run.js --subscriptions $*
8+
else
9+
node ./node_modules/@subsquid/graphql-server/bin/run.js --subscriptions $*
10+
fi

‎opentelemetry/.env‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
# Required env variables for the Elasticsearch exporters
2+
OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200
3+
OTEL_RESOURCE_ATTRIBUTES="service.name=orion-graphql-server,deployment.environment=production"
4+
OTEL_METRICS_EXPORTER="otlp"
5+
6+
# Optional env vars to configure the opentelemetry exporters
7+
OTEL_MAX_QUEUE_SIZE=8192 # 4 times of default queue size
8+
OTEL_MAX_EXPORT_BATCH_SIZE=1024 # 2 times of default batch size

‎opentelemetry/index.js‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
const { DiagConsoleLogger, DiagLogLevel, diag } = require('@opentelemetry/api')
2+
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node')
3+
4+
/**
5+
* Error: "@opentelemetry/instrumentation-grpc Module @grpc/grpc-js has
6+
* been loaded before @opentelemetry/instrumentation-grpc so it might not work,
7+
* please initialize it before requiring @grpc/grpc-js"
8+
*
9+
* Fix: "call getNodeAutoInstrumentations() before require('@opentelemetry/sdk-node');"
10+
*/
11+
//
12+
13+
// Disable DNS instrumentation, because the instrumentation does not correctly patches `dns.lookup` function
14+
// if the function is converted to a promise-based method using `utils.promisify(dns.lookup)`
15+
// See: https://github.com/Joystream/joystream/pull/4779#discussion_r1262515887
16+
const instrumentations = getNodeAutoInstrumentations({
17+
'@opentelemetry/instrumentation-dns': { enabled: false },
18+
'@opentelemetry/instrumentation-pg': { enhancedDatabaseReporting: true },
19+
})
20+
21+
const { NodeSDK } = require('@opentelemetry/sdk-node')
22+
const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-proto')
23+
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-proto')
24+
const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics')
25+
const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-node')
26+
27+
const path = require('path')
28+
require('dotenv').config({ path: path.resolve(__dirname, `.env`) })
29+
30+
// For troubleshooting, set the log level to DiagLogLevel.DEBUG
31+
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO)
32+
33+
function addInstrumentation() {
34+
const instrumentation = new NodeSDK({
35+
spanProcessor: new BatchSpanProcessor(new OTLPTraceExporter(), {
36+
maxQueueSize: parseInt(process.env.OTEL_MAX_QUEUE_SIZE || '8192'),
37+
maxExportBatchSize: parseInt(process.env.OTEL_MAX_EXPORT_BATCH_SIZE || '1024'),
38+
}),
39+
metricReader: new PeriodicExportingMetricReader({
40+
exporter: new OTLPMetricExporter(),
41+
}),
42+
instrumentations,
43+
})
44+
45+
// Start Opentelemetry NodeJS Instrumentation
46+
diag.info('Starting tracing...')
47+
instrumentation.start()
48+
49+
// gracefully shut down the SDK on process exit
50+
process.on('SIGTERM', () => {
51+
instrumentation
52+
.shutdown()
53+
.then(() => console.log('Tracing terminated'))
54+
.catch((error) => console.log('Error terminating tracing', error))
55+
.finally(() => process.exit(0))
56+
})
57+
}
58+
59+
addInstrumentation()

0 commit comments

Comments
 (0)