Skip to content

Commit 8f6e650

Browse files
zeeshanakram3zeeshanakram3
committed
integrate Storage-Squid with Storage Node
1 parent de5166d commit 8f6e650

11 files changed

Lines changed: 1876 additions & 32973 deletions

File tree

‎storage-node/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
"fast-folder-size": "^1.4.0",
4545
"fast-safe-stringify": "^2.1.1",
4646
"file-type": "^16.5.0",
47+
"graphql": "^15.3.0",
4748
"lodash": "^4.17.21",
4849
"mkdirp": "^0.5.1",
4950
"multihashes": "^4.0.2",
@@ -87,6 +88,7 @@
8788
"eslint": "^7.6.0",
8889
"eslint-config-oclif": "^3.1",
8990
"eslint-config-oclif-typescript": "^0.1",
91+
"get-graphql-schema": "^2.1.2",
9092
"globby": "^10",
9193
"json-schema-to-typescript": "^10.1.4",
9294
"mocha": "^5",
@@ -151,12 +153,14 @@
151153
"format": "yarn prettier ./src --write",
152154
"lint": "eslint ./src --ext .ts",
153155
"api:edit": "openapi-editor --file ./src/api-spec/openapi.yaml --port 10021",
156+
"generate:schema:graphql": "../node_modules/get-graphql-schema/dist/index.js http://localhost:4352/graphql > src/services/queryNode/schema.graphql",
154157
"generate:types:graphql": "yarn graphql-codegen -c ./src/services/queryNode/codegen.yml",
155158
"generate:types:json-schema": "yarn ts-node --transpile-only ./src/services/metadata/generateTypes.ts",
156159
"generate:types:api": "yarn openapi-typescript ./src/api-spec/openapi.yaml -o ./src/services/webApi/types/generated.ts -c ../prettierrc.js",
157160
"generate:types:all": "yarn generate:types:graphql && yarn generate:types:json-schema && yarn generate:types:api",
158161
"generate:client": "yarn openapi-generator-cli generate -i ./src/api-spec/openapi.yaml -g typescript-axios -o ./client",
159162
"generate:all": "yarn generate:types:graphql && yarn generate:types:json-schema && yarn generate:types:api && yarn generate:client",
163+
"generate:docs:cli": "yarn oclif-dev readme",
160164
"ensure": "yarn format && yarn lint --fix && yarn build",
161165
"checks": "tsc --noEmit --pretty && prettier ./src --check && yarn lint",
162166
"start": "./bin/run server",

‎storage-node/src/commands/dev/cleanup.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
import { Command, flags } from '@oclif/command'
1+
import { flags } from '@oclif/command'
22
import stringify from 'fast-safe-stringify'
3+
import ApiCommandBase from '../../command-base/ApiCommandBase'
34
import logger from '../../services/logger'
45
import { QueryNodeApi } from '../../services/queryNode/api'
56
import { performCleanup } from '../../services/sync/cleanupService'
@@ -13,11 +14,10 @@ import { performCleanup } from '../../services/sync/cleanupService'
1314
* Should be run only during the development.
1415
* Shell command: "dev:cleanup"
1516
*/
16-
export default class DevCleanup extends Command {
17+
export default class DevCleanup extends ApiCommandBase {
1718
static description = `Runs the data objects cleanup/pruning workflow. It removes all the local stored data objects that the operator is no longer obliged to store`
1819

1920
static flags = {
20-
help: flags.help({ char: 'h' }),
2121
workerId: flags.integer({
2222
char: 'w',
2323
required: true,
@@ -37,24 +37,27 @@ export default class DevCleanup extends Command {
3737
queryNodeEndpoint: flags.string({
3838
char: 'q',
3939
required: false,
40-
default: 'http://localhost:8081/graphql',
41-
description: 'Query node endpoint (e.g.: http://some.com:8081/graphql)',
40+
default: 'http://localhost:4352/graphql',
41+
description: 'Storage Squid graphql server endpoint (e.g.: http://some.com:4352/graphql)',
4242
}),
4343
uploads: flags.string({
4444
char: 'd',
4545
required: true,
4646
description: 'Data uploading directory (absolute path).',
4747
}),
48+
...ApiCommandBase.flags,
4849
}
4950

5051
async run(): Promise<void> {
5152
const { flags } = this.parse(DevCleanup)
5253
const bucketId = flags.bucketId.toString()
54+
const api = await this.getApi()
5355
const qnApi = new QueryNodeApi(flags.queryNodeEndpoint)
56+
5457
logger.info('Cleanup...')
5558

5659
try {
57-
await performCleanup(flags.workerId, [bucketId], flags.cleanupWorkersNumber, qnApi, flags.uploads)
60+
await performCleanup(flags.workerId, [bucketId], flags.cleanupWorkersNumber, api, qnApi, flags.uploads)
5861
} catch (err) {
5962
logger.error(err)
6063
logger.error(stringify(err))

‎storage-node/src/commands/dev/sync.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ export default class DevSync extends Command {
4444
queryNodeEndpoint: flags.string({
4545
char: 'q',
4646
required: false,
47-
default: 'http://localhost:8081/graphql',
48-
description: 'Query node endpoint (e.g.: http://some.com:8081/graphql)',
47+
default: 'http://localhost:4352/graphql',
48+
description: 'Storage Squid graphql server endpoint (e.g.: http://some.com:4352/graphql)',
4949
}),
5050
dataSourceOperatorUrl: flags.string({
5151
char: 'o',

‎storage-node/src/commands/server.ts‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,14 @@ import { customFlags } from '../command-base/CustomFlags'
1313
import { loadDataObjectIdCache } from '../services/caching/localDataObjects'
1414
import logger, { DatePatternByFrequency, Frequency, initNewLogger } from '../services/logger'
1515
import { QueryNodeApi } from '../services/queryNode/api'
16+
import { AcceptPendingObjectsService } from '../services/sync/acceptPendingObjects'
1617
import {
1718
MAXIMUM_QN_LAGGING_THRESHOLD,
1819
MINIMUM_REPLICATION_THRESHOLD,
1920
performCleanup,
2021
} from '../services/sync/cleanupService'
21-
import { AcceptPendingObjectsService } from '../services/sync/acceptPendingObjects'
2222
import { getStorageBucketIdsByWorkerId } from '../services/sync/storageObligations'
23-
import { PendingDirName, performSync, TempDirName } from '../services/sync/synchronizer'
23+
import { PendingDirName, TempDirName, performSync } from '../services/sync/synchronizer'
2424
import { createApp } from '../services/webApi/app'
2525
import ExitCodes from './../command-base/ExitCodes'
2626
const fsPromises = fs.promises
@@ -84,8 +84,8 @@ export default class Server extends ApiCommandBase {
8484
queryNodeEndpoint: flags.string({
8585
char: 'q',
8686
required: true,
87-
default: 'http://localhost:8081/graphql',
88-
description: 'Query node endpoint (e.g.: http://some.com:8081/graphql)',
87+
default: 'http://localhost:4352/graphql',
88+
description: 'Storage Squid graphql server endpoint (e.g.: http://some.com:4352/graphql)',
8989
}),
9090
syncWorkersNumber: flags.integer({
9191
char: 'r',
@@ -278,6 +278,7 @@ Supported values: warn, error, debug, info. Default:debug`,
278278
runCleanupWithInterval(
279279
flags.worker,
280280
selectedBuckets,
281+
api,
281282
qnApi,
282283
flags.uploads,
283284
flags.syncWorkersNumber,
@@ -399,6 +400,7 @@ async function runSyncWithInterval(
399400
async function runCleanupWithInterval(
400401
workerId: number,
401402
buckets: string[],
403+
api: ApiPromise,
402404
qnApi: QueryNodeApi,
403405
uploadsDirectory: string,
404406
syncWorkersNumber: number,
@@ -410,7 +412,7 @@ async function runCleanupWithInterval(
410412
await sleep(sleepInterval)
411413
try {
412414
logger.info(`Resume cleanup....`)
413-
await performCleanup(workerId, buckets, syncWorkersNumber, qnApi, uploadsDirectory)
415+
await performCleanup(workerId, buckets, syncWorkersNumber, api, qnApi, uploadsDirectory)
414416
} catch (err) {
415417
logger.error(`Critical cleanup error: ${err}`)
416418
}

0 commit comments

Comments
 (0)