Skip to content
This repository was archived by the owner on Oct 15, 2024. It is now read-only.

Commit 5736c74

Browse files
opaolinidekz
andauthored
feat: Graceful Shutdowns (#280)
* init graceful shutdowns * fix test utils regex * handle async close * add healtcheck port configuration, and statuscode to response * move server listening info, add SIGQUIT * Update src/config.ts Co-authored-by: Jacob Evans <jacob@dekz.net> * reorder on close callback Co-authored-by: Jacob Evans <jacob@dekz.net>
1 parent 456b4fd commit 5736c74

13 files changed

Lines changed: 163 additions & 61 deletions

src/config.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@ export const HTTP_PORT = _.isEmpty(process.env.HTTP_PORT)
6262
? 3000
6363
: assertEnvVarType('HTTP_PORT', process.env.HTTP_PORT, EnvVarType.Port);
6464

65+
// Network port for the healthcheck service at /healthz, if not provided, it uses the HTTP_PORT value.
66+
export const HEALTHCHECK_HTTP_PORT = _.isEmpty(process.env.HEALTHCHECK_HTTP_PORT)
67+
? HTTP_PORT
68+
: assertEnvVarType('HEALTHCHECK_HTTP_PORT', process.env.HEALTHCHECK_HTTP_PORT, EnvVarType.Port);
69+
6570
// Number of milliseconds of inactivity the servers waits for additional
6671
// incoming data aftere it finished writing last response before a socket will
6772
// be destroyed.
@@ -379,6 +384,7 @@ export const SWAP_QUOTER_OPTS: Partial<SwapQuoterOpts> = {
379384

380385
export const defaultHttpServiceConfig: HttpServiceConfig = {
381386
httpPort: HTTP_PORT,
387+
healthcheckHttpPort: HEALTHCHECK_HTTP_PORT,
382388
ethereumRpcUrl: ETHEREUM_RPC_URL,
383389
httpKeepAliveTimeout: HTTP_KEEP_ALIVE_TIMEOUT,
384390
httpHeadersTimeout: HTTP_HEADERS_TIMEOUT,

src/constants.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ export const SWAP_PATH = '/swap';
4949
export const META_TRANSACTION_PATH = '/meta_transaction/v0';
5050
export const METRICS_PATH = '/metrics';
5151
export const API_KEY_HEADER = '0x-api-key';
52+
export const HEALTHCHECK_PATH = '/healthz';
5253

5354
// Docs
5455
export const SWAP_DOCS_URL = 'https://0x.org/docs/api#swap';
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import * as express from 'express';
2+
import * as HttpStatus from 'http-status-codes';
3+
4+
import { HealthcheckService } from '../services/healthcheck_service';
5+
6+
export class HealthcheckHandlers {
7+
private readonly _healthcheckService: HealthcheckService;
8+
9+
constructor(healthcheckService: HealthcheckService) {
10+
this._healthcheckService = healthcheckService;
11+
}
12+
13+
public serveHealthcheck(_req: express.Request, res: express.Response): void {
14+
const isHealthy = this._healthcheckService.isHealthy();
15+
if (isHealthy) {
16+
res.status(HttpStatus.OK).send({ isHealthy });
17+
} else {
18+
res.status(HttpStatus.SERVICE_UNAVAILABLE).send({ isHealthy });
19+
}
20+
}
21+
}

src/routers/healthcheck_router.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
import * as express from 'express';
2+
3+
import { HealthcheckHandlers } from '../handlers/healthcheck_handlers';
4+
import { HealthcheckService } from '../services/healthcheck_service';
5+
6+
export const createHealthcheckRouter = (healthcheckService: HealthcheckService): express.Router => {
7+
const router = express.Router();
8+
const handlers = new HealthcheckHandlers(healthcheckService);
9+
/**
10+
* GET healthcheck endpoint returns the health of the http server.
11+
*/
12+
router.get('/', handlers.serveHealthcheck.bind(handlers));
13+
return router;
14+
};

src/runners/http_meta_transaction_service_runner.ts

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import * as bodyParser from 'body-parser';
2-
import * as cors from 'cors';
31
import * as express from 'express';
42
// tslint:disable-next-line:no-implicit-dependencies
53
import * as core from 'express-serve-static-core';
@@ -11,11 +9,12 @@ import { META_TRANSACTION_PATH } from '../constants';
119
import { rootHandler } from '../handlers/root_handler';
1210
import { logger } from '../logger';
1311
import { errorHandler } from '../middleware/error_handling';
14-
import { requestLogger } from '../middleware/request_logger';
1512
import { createMetaTransactionRouter } from '../routers/meta_transaction_router';
1613
import { HttpServiceConfig } from '../types';
1714
import { providerUtils } from '../utils/provider_utils';
1815

16+
import { createDefaultServer } from './utils';
17+
1918
/**
2019
* This module can be used to run the Meta Transaction HTTP service standalone
2120
*/
@@ -45,14 +44,8 @@ async function runHttpServiceAsync(
4544
_app?: core.Express,
4645
): Promise<Server> {
4746
const app = _app || express();
48-
app.use(requestLogger());
49-
app.use(cors());
50-
app.use(bodyParser.json());
47+
const server = createDefaultServer(dependencies, config, app);
5148
app.get('/', rootHandler);
52-
const server = app.listen(config.httpPort, () => {
53-
logger.info(`API (HTTP) listening on port ${config.httpPort}!`);
54-
});
55-
5649
if (dependencies.metaTransactionService) {
5750
app.use(
5851
META_TRANSACTION_PATH,
@@ -63,5 +56,6 @@ async function runHttpServiceAsync(
6356
process.exit(1);
6457
}
6558
app.use(errorHandler);
59+
server.listen(config.httpPort);
6660
return server;
6761
}

src/runners/http_service_runner.ts

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import bodyParser = require('body-parser');
2-
import * as cors from 'cors';
31
import * as express from 'express';
42
// tslint:disable-next-line:no-implicit-dependencies
53
import * as core from 'express-serve-static-core';
@@ -12,7 +10,6 @@ import { rootHandler } from '../handlers/root_handler';
1210
import { logger } from '../logger';
1311
import { addressNormalizer } from '../middleware/address_normalizer';
1412
import { errorHandler } from '../middleware/error_handling';
15-
import { requestLogger } from '../middleware/request_logger';
1613
import { createMetaTransactionRouter } from '../routers/meta_transaction_router';
1714
import { createMetricsRouter } from '../routers/metrics_router';
1815
import { createSRARouter } from '../routers/sra_router';
@@ -22,6 +19,8 @@ import { WebsocketService } from '../services/websocket_service';
2219
import { HttpServiceConfig } from '../types';
2320
import { providerUtils } from '../utils/provider_utils';
2421

22+
import { createDefaultServer } from './utils';
23+
2524
/**
2625
* http_service_runner hosts endpoints for staking, sra, swap and meta-txns (minus the /submit endpoint)
2726
* and can be horizontally scaled as needed
@@ -63,19 +62,12 @@ export async function runHttpServiceAsync(
6362
_app?: core.Express,
6463
): Promise<HttpServices> {
6564
const app = _app || express();
66-
app.use(requestLogger());
67-
app.use(cors());
68-
app.use(bodyParser.json());
65+
const server = createDefaultServer(dependencies, config, app);
6966

7067
app.get('/', rootHandler);
71-
const server = app.listen(config.httpPort, () => {
72-
logger.info(`API (HTTP) listening on port ${config.httpPort}!`);
73-
});
7468
server.on('error', err => {
7569
logger.error(err);
7670
});
77-
server.keepAliveTimeout = config.httpKeepAliveTimeout;
78-
server.headersTimeout = config.httpHeadersTimeout;
7971

8072
// transform all values of `req.query.[xx]Address` to lowercase
8173
app.use(addressNormalizer);
@@ -132,7 +124,7 @@ export async function runHttpServiceAsync(
132124
logger.error(`Could not establish mesh connection, exiting`);
133125
process.exit(1);
134126
}
135-
127+
server.listen(config.httpPort);
136128
return {
137129
server,
138130
wsService,

src/runners/http_sra_service_runner.ts

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
/**
22
* This module can be used to run the SRA HTTP service standalone
33
*/
4-
5-
import bodyParser = require('body-parser');
6-
import * as cors from 'cors';
74
import * as express from 'express';
85
// tslint:disable-next-line:no-implicit-dependencies
96
import * as core from 'express-serve-static-core';
@@ -16,12 +13,13 @@ import { rootHandler } from '../handlers/root_handler';
1613
import { logger } from '../logger';
1714
import { addressNormalizer } from '../middleware/address_normalizer';
1815
import { errorHandler } from '../middleware/error_handling';
19-
import { requestLogger } from '../middleware/request_logger';
2016
import { createSRARouter } from '../routers/sra_router';
2117
import { WebsocketService } from '../services/websocket_service';
2218
import { HttpServiceConfig } from '../types';
2319
import { providerUtils } from '../utils/provider_utils';
2420

21+
import { createDefaultServer } from './utils';
22+
2523
process.on('uncaughtException', err => {
2624
logger.error(err);
2725
process.exit(1);
@@ -47,18 +45,11 @@ async function runHttpServiceAsync(
4745
_app?: core.Express,
4846
): Promise<Server> {
4947
const app = _app || express();
50-
app.use(requestLogger());
51-
app.use(cors());
52-
app.use(bodyParser.json());
53-
app.get('/', rootHandler);
54-
const server = app.listen(config.httpPort, () => {
55-
logger.info(`API (HTTP) listening on port ${config.httpPort}!`);
56-
});
57-
server.keepAliveTimeout = config.httpKeepAliveTimeout;
58-
server.headersTimeout = config.httpHeadersTimeout;
48+
app.use(addressNormalizer);
49+
const server = createDefaultServer(dependencies, config, app);
5950

51+
app.get('/', rootHandler);
6052
// SRA http service
61-
app.use(addressNormalizer);
6253
app.use(SRA_PATH, createSRARouter(dependencies.orderBookService));
6354
app.use(errorHandler);
6455

@@ -71,5 +62,6 @@ async function runHttpServiceAsync(
7162
process.exit(1);
7263
}
7364

65+
server.listen(config.httpPort);
7466
return server;
7567
}

src/runners/http_staking_service_runner.ts

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
/**
22
* This module can be used to run the Staking HTTP service standalone
33
*/
4-
5-
import bodyParser = require('body-parser');
6-
import * as cors from 'cors';
74
import * as express from 'express';
85
// tslint:disable-next-line:no-implicit-dependencies
96
import * as core from 'express-serve-static-core';
@@ -16,11 +13,12 @@ import { rootHandler } from '../handlers/root_handler';
1613
import { logger } from '../logger';
1714
import { addressNormalizer } from '../middleware/address_normalizer';
1815
import { errorHandler } from '../middleware/error_handling';
19-
import { requestLogger } from '../middleware/request_logger';
2016
import { createStakingRouter } from '../routers/staking_router';
2117
import { HttpServiceConfig } from '../types';
2218
import { providerUtils } from '../utils/provider_utils';
2319

20+
import { createDefaultServer } from './utils';
21+
2422
process.on('uncaughtException', err => {
2523
logger.error(err);
2624
process.exit(1);
@@ -46,19 +44,14 @@ async function runHttpServiceAsync(
4644
_app?: core.Express,
4745
): Promise<Server> {
4846
const app = _app || express();
49-
app.use(requestLogger());
50-
app.use(cors());
51-
app.use(bodyParser.json());
5247
app.use(addressNormalizer);
53-
app.get('/', rootHandler);
54-
const server = app.listen(config.httpPort, () => {
55-
logger.info(`API (HTTP) listening on port ${config.httpPort}!`);
56-
});
57-
server.keepAliveTimeout = config.httpKeepAliveTimeout;
58-
server.headersTimeout = config.httpHeadersTimeout;
48+
const server = createDefaultServer(dependencies, config, app);
5949

50+
app.get('/', rootHandler);
6051
// staking http service
6152
app.use(STAKING_PATH, createStakingRouter(dependencies.stakingDataService));
6253
app.use(errorHandler);
54+
55+
server.listen(config.httpPort);
6356
return server;
6457
}

src/runners/http_swap_service_runner.ts

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22
* This module can be used to run the Swap HTTP service standalone
33
*/
44

5-
import bodyParser = require('body-parser');
6-
import * as cors from 'cors';
75
import * as express from 'express';
86
// tslint:disable-next-line:no-implicit-dependencies
97
import * as core from 'express-serve-static-core';
@@ -16,11 +14,12 @@ import { rootHandler } from '../handlers/root_handler';
1614
import { logger } from '../logger';
1715
import { addressNormalizer } from '../middleware/address_normalizer';
1816
import { errorHandler } from '../middleware/error_handling';
19-
import { requestLogger } from '../middleware/request_logger';
2017
import { createSwapRouter } from '../routers/swap_router';
2118
import { HttpServiceConfig } from '../types';
2219
import { providerUtils } from '../utils/provider_utils';
2320

21+
import { createDefaultServer } from './utils';
22+
2423
process.on('uncaughtException', err => {
2524
logger.error(err);
2625
process.exit(1);
@@ -46,16 +45,10 @@ async function runHttpServiceAsync(
4645
_app?: core.Express,
4746
): Promise<Server> {
4847
const app = _app || express();
49-
app.use(requestLogger());
50-
app.use(cors());
51-
app.use(bodyParser.json());
5248
app.use(addressNormalizer);
49+
const server = createDefaultServer(dependencies, config, app);
50+
5351
app.get('/', rootHandler);
54-
const server = app.listen(config.httpPort, () => {
55-
logger.info(`API (HTTP) listening on port ${config.httpPort}!`);
56-
});
57-
server.keepAliveTimeout = config.httpKeepAliveTimeout;
58-
server.headersTimeout = config.httpHeadersTimeout;
5952

6053
if (dependencies.swapService) {
6154
app.use(SWAP_PATH, createSwapRouter(dependencies.swapService));
@@ -64,5 +57,7 @@ async function runHttpServiceAsync(
6457
process.exit(1);
6558
}
6659
app.use(errorHandler);
60+
61+
server.listen(config.httpPort);
6762
return server;
6863
}

src/runners/utils.ts

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import * as bodyParser from 'body-parser';
2+
import * as cors from 'cors';
3+
import * as express from 'express';
4+
// tslint:disable-next-line:no-implicit-dependencies
5+
import * as core from 'express-serve-static-core';
6+
import { createServer, Server } from 'http';
7+
8+
import { AppDependencies } from '../app';
9+
import { HEALTHCHECK_PATH } from '../constants';
10+
import { logger } from '../logger';
11+
import { requestLogger } from '../middleware/request_logger';
12+
import { createHealthcheckRouter } from '../routers/healthcheck_router';
13+
import { HealthcheckService } from '../services/healthcheck_service';
14+
import { HttpServiceConfig } from '../types';
15+
16+
/**
17+
* creates the NodeJS http server with graceful shutdowns, healthchecks,
18+
* configured header timeouts and other sane defaults set.
19+
*/
20+
export function createDefaultServer(
21+
dependencies: AppDependencies,
22+
config: HttpServiceConfig,
23+
app: core.Express,
24+
): Server {
25+
app.use(requestLogger());
26+
app.use(cors());
27+
app.use(bodyParser.json());
28+
29+
const server = createServer(app);
30+
server.keepAliveTimeout = config.httpKeepAliveTimeout;
31+
server.headersTimeout = config.httpHeadersTimeout;
32+
const healthcheckService = new HealthcheckService();
33+
34+
server.on('close', () => {
35+
logger.info('http server shutdown');
36+
});
37+
server.on('listening', () => {
38+
logger.info(`server listening on ${config.httpPort}`);
39+
healthcheckService.setHealth(true);
40+
});
41+
42+
const shutdownFunc = (sig: string) => {
43+
logger.info(`received: ${sig}, shutting down server`);
44+
healthcheckService.setHealth(false);
45+
server.close(async err => {
46+
if (dependencies.meshClient) {
47+
dependencies.meshClient.destroy();
48+
}
49+
if (dependencies.connection) {
50+
await dependencies.connection.close();
51+
}
52+
if (!server.listening) {
53+
process.exit(0);
54+
}
55+
if (err) {
56+
logger.error(`server closed with an error: ${err}, exiting`);
57+
process.exit(1);
58+
}
59+
logger.info('successful shutdown, exiting');
60+
process.exit(0);
61+
});
62+
};
63+
if (config.httpPort === config.healthcheckHttpPort) {
64+
app.use(HEALTHCHECK_PATH, createHealthcheckRouter(healthcheckService));
65+
} else {
66+
// if we don't want to expose the /healthz healthcheck service route to
67+
// the public, we serve it from a different port. Serving it through a
68+
// different express app also removes the unnecessary request logging.
69+
const healthcheckApp = express();
70+
healthcheckApp.use(HEALTHCHECK_PATH, createHealthcheckRouter(healthcheckService));
71+
healthcheckApp.listen(config.healthcheckHttpPort, () => {
72+
logger.info(`healthcheckApp listening on ${config.healthcheckHttpPort}`);
73+
});
74+
}
75+
process.on('SIGINT', shutdownFunc);
76+
process.on('SIGTERM', shutdownFunc);
77+
process.on('SIGQUIT', shutdownFunc);
78+
return server;
79+
}

0 commit comments

Comments
 (0)