Skip to content

Commit 73b7820

Browse files
mykola-mokhnachmykola-mokhnach
authored
feat(strongbox): Add a possibility to list strongbox items (#22091)
1 parent 26f8429 commit 73b7820

6 files changed

Lines changed: 283 additions & 7 deletions

File tree

‎packages/strongbox/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,20 @@ Or write new data to the item:
4949
await item.write('new stuff');
5050
```
5151

52+
To list persisted items without knowing their names in advance:
53+
54+
```ts
55+
const items = await box.listItems();
56+
```
57+
58+
`listItems` does not read each file’s contents; call `read()` on an item when you need them. Item order follows the filesystem directory walk (not lexicographic). It returns every item in one array; for large containers that can use a lot of memory, prefer async iteration, which streams directory entries with `opendir` instead of buffering all names first:
59+
60+
```ts
61+
for await (const item of box) {
62+
// ...
63+
}
64+
```
65+
5266
The last-read contents of the `Item` will be available on the `contents` property, but the value of this property is only current as of the last `read()`:
5367

5468
```ts

‎packages/strongbox/lib/base-item.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,14 @@ import {slugify} from './util';
1010
* @typeParam T - Type of data stored in the `Item`
1111
*/
1212
export class BaseItem<T extends Value, U extends Strongbox = Strongbox> implements Item<T> {
13+
/**
14+
* Absolute filesystem path of the file backing an item: `container` + slugified `name`.
15+
* Also used to convert a `name` to an `id`.
16+
*/
17+
public static toFilePath(container: string, name: string): string {
18+
return path.join(container, slugify(name));
19+
}
20+
1321
/**
1422
* {@inheritdoc Item.value}
1523
*/
@@ -40,7 +48,7 @@ export class BaseItem<T extends Value, U extends Strongbox = Strongbox> implemen
4048
public readonly encoding: ItemEncoding = 'utf8'
4149
) {
4250
this.container = parent.container;
43-
this.id = path.join(this.container, slugify(name));
51+
this.id = BaseItem.toFilePath(this.container, name);
4452
Object.defineProperties(this, {
4553
value: {
4654
get() {

‎packages/strongbox/lib/index.ts‎

Lines changed: 96 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import envPaths from 'env-paths';
2-
import {rm} from 'node:fs/promises';
2+
import {opendir, rm} from 'node:fs/promises';
33
import path from 'node:path';
44
import {BaseItem} from './base-item';
55
import {slugify} from './util';
@@ -106,7 +106,7 @@ export type ItemCtor<
106106
*
107107
* Manages multiple {@linkcode Item}s.
108108
*/
109-
export class Strongbox<Options extends StrongboxOpts = StrongboxOpts> {
109+
export class Strongbox<Options extends StrongboxOpts = StrongboxOpts> implements AsyncIterable<Item<any>> {
110110
/**
111111
* Default {@linkcode ItemCtor} to use when creating new {@linkcode Item}s
112112
*/
@@ -204,7 +204,7 @@ export class Strongbox<Options extends StrongboxOpts = StrongboxOpts> {
204204
this,
205205
encoding
206206
);
207-
if (this.items.has(item.id)) {
207+
if (this.getLiveItem(item.id)) {
208208
throw new ReferenceError(`Item with id "${item.id}" already exists`);
209209
}
210210
try {
@@ -254,12 +254,64 @@ export class Strongbox<Options extends StrongboxOpts = StrongboxOpts> {
254254

255255
/**
256256
* Attempts to retrieve an {@linkcode Item} by its `id`.
257+
* Drops stale {@linkcode WeakRef} map entries when the value was collected.
257258
* @param id ID of item
258259
* @returns An `Item`, if found
259260
*/
260261
public getItem(id: string): Item<any> | undefined {
262+
return this.getLiveItem(id);
263+
}
264+
265+
/**
266+
* Returns a live {@linkcode Item} or removes a stale {@linkcode WeakRef} from {@linkcode Strongbox.items}.
267+
*/
268+
private getLiveItem(id: string): Item<any> | undefined {
261269
const ref = this.items.get(id);
262-
return ref?.deref();
270+
if (!ref) {
271+
return undefined;
272+
}
273+
const item = ref.deref();
274+
if (!item) {
275+
this.items.delete(id);
276+
return undefined;
277+
}
278+
return item;
279+
}
280+
281+
/**
282+
* Lists persisted items by scanning the container directory (one regular file per item).
283+
*
284+
* Filenames are matched to items by path; if an item was already registered (e.g. via
285+
* {@linkcode Strongbox.createItem}), that instance is returned and keeps its original `name`.
286+
* Otherwise a new item is created using the filename as `name` (see {@linkcode BaseItem}).
287+
*
288+
* @remarks Builds one array of every {@linkcode Item} reference. Does not read file contents;
289+
* call {@linkcode Item.read} on each item as needed. Order follows directory iteration
290+
* ({@linkcode opendir}), not lexicographic sort. For many items, `for await (const item of box)`
291+
* ({@linkcode Symbol.asyncIterator}) streams entries without allocating a full {@linkcode Item}[]
292+
* first.
293+
*
294+
* @returns Items in directory iteration order; empty if the container directory does not exist yet
295+
*/
296+
public async listItems(): Promise<Item<any>[]> {
297+
const items: Item<any>[] = [];
298+
for await (const basename of this.iterateFileBasenames()) {
299+
items.push(this.resolveItemForBasename(basename));
300+
}
301+
return items;
302+
}
303+
304+
/**
305+
* Yields each persisted item in the same order as {@linkcode Strongbox.listItems}. Use
306+
* `for await (const item of box)`.
307+
*
308+
* @remarks Walks the container with {@linkcode opendir} and yields one {@linkcode Item} per file
309+
* as basenames are seen (no full-name buffer and no sort).
310+
*/
311+
public async *[Symbol.asyncIterator](): AsyncIterableIterator<Item<any>> {
312+
for await (const basename of this.iterateFileBasenames()) {
313+
yield this.resolveItemForBasename(basename);
314+
}
263315
}
264316

265317
/**
@@ -298,6 +350,46 @@ export class Strongbox<Options extends StrongboxOpts = StrongboxOpts> {
298350
newOpts.defaultItemCtor = opts.defaultItemCtor ?? BaseItem;
299351
return newOpts;
300352
}
353+
354+
/**
355+
* Streams regular-file basenames from the container using {@linkcode opendir} (order is
356+
* filesystem-defined, not sorted).
357+
*/
358+
private async *iterateFileBasenames(): AsyncIterableIterator<string> {
359+
let dir: Awaited<ReturnType<typeof opendir>>;
360+
try {
361+
dir = await opendir(this.container);
362+
} catch (e) {
363+
if ((e as NodeJS.ErrnoException).code === 'ENOENT') {
364+
return;
365+
}
366+
throw e;
367+
}
368+
for await (const ent of dir) {
369+
if (ent.isFile()) {
370+
yield ent.name;
371+
}
372+
}
373+
}
374+
375+
/**
376+
* Registers an {@linkcode Item} for a filename on disk without reading the file (see
377+
* {@linkcode Strongbox.createItem}, which loads persisted contents eagerly). Uses the same
378+
* constructor arity as {@linkcode Strongbox.createItem} when no encoding is given.
379+
*/
380+
private registerItemWithoutRead(name: string): Item<any> {
381+
const item = new (this.defaultItemCtor as ItemCtor<any>)(name, this, undefined);
382+
if (this.getLiveItem(item.id)) {
383+
throw new ReferenceError(`Item with id "${item.id}" already exists`);
384+
}
385+
this.items.set(item.id, new WeakRef(item));
386+
return item;
387+
}
388+
389+
private resolveItemForBasename(basename: string): Item<any> {
390+
const id = BaseItem.toFilePath(this.container, basename);
391+
return this.getItem(id) ?? this.registerItemWithoutRead(basename);
392+
}
301393
}
302394

303395
/**

‎packages/strongbox/test/e2e/strongbox.e2e.spec.ts‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import {readFile} from 'node:fs/promises';
1+
import {readFile, rm} from 'node:fs/promises';
22
import type {Item, Strongbox} from '../../lib';
33
import {strongbox} from '../../lib';
44

@@ -21,7 +21,7 @@ describe('@appium/strongbox', function () {
2121
});
2222

2323
afterEach(async function () {
24-
await box.clearAll(true);
24+
await rm(box.container, {recursive: true, force: true});
2525
});
2626

2727
describe('when creating an Item with a value', function () {
@@ -77,5 +77,71 @@ describe('@appium/strongbox', function () {
7777
});
7878
});
7979
});
80+
81+
describe('listItems()', function () {
82+
it('should return an Item for each persisted file with readable contents', async function () {
83+
await box.createItemWithValue('first', 'a');
84+
await box.createItemWithValue('second item', 'b');
85+
const items = await box.listItems();
86+
expect(items.map((i) => i.name)).to.have.members(['first', 'second item']);
87+
const byName = Object.fromEntries(items.map((i) => [i.name, i]));
88+
await expect(byName.first.read()).to.eventually.equal('a');
89+
await expect(byName['second item'].read()).to.eventually.equal('b');
90+
});
91+
92+
it('should not load persisted contents until read', async function () {
93+
const name = 'e2e-lazy-list';
94+
const writer = strongbox(name);
95+
await rm(writer.container, {recursive: true, force: true});
96+
await writer.createItemWithValue('key', 'payload');
97+
const reader = strongbox(name);
98+
const items = await reader.listItems();
99+
expect(items).to.have.length(1);
100+
expect(items[0].value).to.be.undefined;
101+
await expect(items[0].read()).to.eventually.equal('payload');
102+
await rm(writer.container, {recursive: true, force: true});
103+
});
104+
});
105+
106+
describe('Symbol.asyncIterator', function () {
107+
it('should yield the same Items in the same order as listItems()', async function () {
108+
await box.createItemWithValue('first', 'a');
109+
await box.createItemWithValue('second item', 'b');
110+
const listed = await box.listItems();
111+
const iterated: typeof listed = [];
112+
for await (const item of box) {
113+
iterated.push(item);
114+
}
115+
expect(iterated.map((i) => i.name)).to.eql(listed.map((i) => i.name));
116+
expect(iterated).to.eql(listed);
117+
});
118+
});
119+
120+
describe('persistence across Strongbox instances', function () {
121+
const NAME = 'e2e-persistence-instance';
122+
123+
beforeEach(async function () {
124+
await rm(strongbox(NAME).container, {recursive: true, force: true});
125+
});
126+
127+
afterEach(async function () {
128+
await rm(strongbox(NAME).container, {recursive: true, force: true});
129+
});
130+
131+
it('should expose persisted items from a second instance with the same identifier', async function () {
132+
const first = strongbox(NAME);
133+
await first.createItemWithValue('item-a', 'hello');
134+
await first.createItemWithValue('item-b', 'world');
135+
136+
const second = strongbox(NAME);
137+
expect(second.container).to.equal(first.container);
138+
139+
const items = await second.listItems();
140+
expect(items.map((i) => i.name)).to.have.members(['item-a', 'item-b']);
141+
const byName = Object.fromEntries(items.map((i) => [i.name, i]));
142+
await expect(byName['item-a'].read()).to.eventually.equal('hello');
143+
await expect(byName['item-b'].read()).to.eventually.equal('world');
144+
});
145+
});
80146
});
81147
});

‎packages/strongbox/test/types/strongbox.test-d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import {expectAssignable, expectNotAssignable} from 'tsd';
22
import {Item, BaseItem, Value, strongbox} from '../..';
33

44
expectAssignable<Item<string>>(new BaseItem('foo', strongbox('foo')));
5+
expectAssignable<AsyncIterable<Item<any>>>(strongbox('foo'));
56

67
expectNotAssignable<Value>(1);
78
expectAssignable<Value>('foo');

‎packages/strongbox/test/unit/strongbox.spec.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,101 @@ describe('Strongbox', function () {
233233
});
234234
});
235235
});
236+
237+
describe('listItems()', function () {
238+
function dirent(name: string, file = true) {
239+
return {
240+
name,
241+
isFile: () => file,
242+
isDirectory: () => !file,
243+
};
244+
}
245+
246+
function mockOpendir(entries: any[]) {
247+
MockFs.opendir.resolves({
248+
async *[Symbol.asyncIterator]() {
249+
for (const e of entries) {
250+
yield e;
251+
}
252+
},
253+
close: sandbox.stub().resolves(),
254+
} as any);
255+
}
256+
257+
it('should return Items for each file in opendir iteration order', async function () {
258+
mockOpendir([dirent('zebra'), dirent('alpha'), dirent('nested', false)]);
259+
const items = await box.listItems();
260+
expect(items.map((i) => i.name)).to.eql(['zebra', 'alpha']);
261+
expect(MockFs.opendir.calledWith(box.container)).to.be.true;
262+
});
263+
264+
it('should return an empty array when the container does not exist', async function () {
265+
const err = Object.assign(new Error('ENOENT'), {code: 'ENOENT'});
266+
MockFs.opendir.rejects(err);
267+
await expect(box.listItems()).to.eventually.eql([]);
268+
});
269+
270+
it('should rethrow non-ENOENT errors', async function () {
271+
MockFs.opendir.rejects(new Error('EACCES'));
272+
await expect(box.listItems()).to.be.rejectedWith('EACCES');
273+
});
274+
275+
it('should reuse an Item already registered on this instance', async function () {
276+
const existing = await box.createItem('alpha');
277+
mockOpendir([dirent('alpha'), dirent('zebra')]);
278+
const items = await box.listItems();
279+
expect(items[0]).to.equal(existing);
280+
expect(items.map((i) => i.name)).to.eql(['alpha', 'zebra']);
281+
});
282+
});
283+
284+
describe('Symbol.asyncIterator', function () {
285+
function dirent(name: string, file = true) {
286+
return {
287+
name,
288+
isFile: () => file,
289+
isDirectory: () => !file,
290+
};
291+
}
292+
293+
function mockOpendir(entries: any[]) {
294+
MockFs.opendir.resolves({
295+
async *[Symbol.asyncIterator]() {
296+
for (const e of entries) {
297+
yield e;
298+
}
299+
},
300+
close: sandbox.stub().resolves(),
301+
} as any);
302+
}
303+
304+
it('should yield the same Items in the same order as listItems()', async function () {
305+
mockOpendir([dirent('zebra'), dirent('alpha'), dirent('nested', false)]);
306+
const fromList = await box.listItems();
307+
const fromIter: typeof fromList = [];
308+
for await (const item of box) {
309+
fromIter.push(item);
310+
}
311+
expect(fromIter.map((i) => i.name)).to.eql(fromList.map((i) => i.name));
312+
expect(fromIter).to.eql(fromList);
313+
});
314+
315+
it('should yield nothing when the container does not exist', async function () {
316+
const err = Object.assign(new Error('ENOENT'), {code: 'ENOENT'});
317+
MockFs.opendir.rejects(err);
318+
const out: any[] = [];
319+
for await (const item of box) {
320+
out.push(item);
321+
}
322+
expect(out).to.eql([]);
323+
});
324+
325+
it('should rethrow non-ENOENT errors', async function () {
326+
MockFs.opendir.rejects(new Error('EACCES'));
327+
const gen = box[Symbol.asyncIterator]();
328+
await expect(gen.next()).to.be.rejectedWith('EACCES');
329+
});
330+
});
236331
});
237332

238333
afterEach(function () {

0 commit comments

Comments
 (0)