Skip to content

Commit d60757e

Browse files
thiyaguk09pearigeeDhriti07
authored
feat(storage): implement Object Contexts with advanced filtering and validation (#7548)
* feat(storage): add Object Contexts support to GCS metadata and listing Updated internal request mapping in `file.ts` and `bucket.ts` to include `contexts` in JSON payloads and `filter` in query strings. Fixed baseline unit tests to accommodate the updated destination metadata structure. * feat(storage): implement Object Contexts with filtering and validation - Add support for Object Contexts metadata in `File` and `Bucket` operations. - Update `FileMetadata`, `CopyOptions`, and `CombineOptions` types to allow null values for context key deletion (PATCH semantics). - Refactor `validateContexts` to accept a `contexts` object directly for better consistency and simpler call patterns in `save`, `copy`, and `combine`. - Implement server-side list filtering support via the `filter` query parameter in `getFiles`, supporting NOT logic and existence wildcards. - Ensure metadata inheritance and explicit overrides work correctly during `File.copy` and `Bucket.combine`. - Add comprehensive unit and system tests covering CRUD, server-side operations, and complex filtering scenarios. * docs(storage): document getFiles filtering and refactor validation logic Extracted context validation into a shared helper for consistency and updated getFiles JSDoc to include Object Context filter syntax and examples. --------- Co-authored-by: Gabe Pearhill <86282859+pearigee@users.noreply.github.com> Co-authored-by: Dhriti07 <56169283+Dhriti07@users.noreply.github.com>
1 parent ecac20f commit d60757e

6 files changed

Lines changed: 654 additions & 2 deletions

File tree

‎handwritten/storage/src/bucket.ts‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import * as path from 'path';
3434
import pLimit from 'p-limit';
3535
import {promisify} from 'util';
3636
import AsyncRetry from 'async-retry';
37-
import {convertObjKeysToSnakeCase} from './util.js';
37+
import {convertObjKeysToSnakeCase, handleContextValidation} from './util.js';
3838

3939
import {Acl, AclMetadata} from './acl.js';
4040
import {Channel} from './channel.js';
@@ -44,6 +44,7 @@ import {
4444
CreateResumableUploadOptions,
4545
CreateWriteStreamOptions,
4646
FileMetadata,
47+
ContextValue,
4748
} from './file.js';
4849
import {Iam} from './iam.js';
4950
import {Notification, NotificationMetadata} from './notification.js';
@@ -178,11 +179,17 @@ export interface GetFilesOptions {
178179
userProject?: string;
179180
versions?: boolean;
180181
fields?: string;
182+
filter?: string;
181183
}
182184

183185
export interface CombineOptions extends PreconditionOptions {
184186
kmsKeyName?: string;
185187
userProject?: string;
188+
contexts?: {
189+
custom: {
190+
[key: string]: ContextValue;
191+
} | null;
192+
};
186193
}
187194

188195
export interface CombineCallback {
@@ -1654,6 +1661,14 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
16541661
options = optionsOrCallback;
16551662
}
16561663

1664+
if (options.contexts) {
1665+
const validationError = handleContextValidation(
1666+
options.contexts,
1667+
callback
1668+
);
1669+
if (validationError) return validationError;
1670+
}
1671+
16571672
this.disableAutoRetryConditionallyIdempotent_(
16581673
this.methods.setMetadata, // Not relevant but param is required
16591674
AvailableServiceObjectMethods.setMetadata, // Same as above
@@ -1708,6 +1723,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
17081723
destination: {
17091724
contentType: destinationFile.metadata.contentType,
17101725
contentEncoding: destinationFile.metadata.contentEncoding,
1726+
contexts: options.contexts || destinationFile.metadata.contexts,
17111727
},
17121728
sourceObjects: (sources as File[]).map(source => {
17131729
const sourceObject = {
@@ -2673,6 +2689,10 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
26732689
* in addition to the relevant part of the object name appearing in prefixes[].
26742690
* @property {string} [prefix] Filter results to objects whose names begin
26752691
* with this prefix.
2692+
* @property {string} [filter] Filter results using a server-side filter
2693+
* expression. This is primarily used for filtering by Object Contexts.
2694+
* Syntax: `contexts."<key>"="<value>"` or `contexts."<key>":*`.
2695+
* Prepend `-` for negation (e.g., `-contexts."key":*`).
26762696
* @property {string} [matchGlob] A glob pattern used to filter results,
26772697
* for example foo*bar
26782698
* @property {number} [maxApiCalls] Maximum number of API calls to make.
@@ -2720,6 +2740,9 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
27202740
* in addition to the relevant part of the object name appearing in prefixes[].
27212741
* @param {string} [query.prefix] Filter results to objects whose names begin
27222742
* with this prefix.
2743+
* @param {string} [query.filter] Filter results using a server-side filter
2744+
* expression. Supports Object Contexts with operators like `=`, `:`,
2745+
* and `-` for negation.
27232746
* @param {number} [query.maxApiCalls] Maximum number of API calls to make.
27242747
* @param {number} [query.maxResults] Maximum number of items plus prefixes to
27252748
* return per call.
@@ -2739,6 +2762,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
27392762
* billed for the request.
27402763
* @param {boolean} [query.versions] If true, returns File objects scoped to
27412764
* their versions.
2765+
*
27422766
* @param {GetFilesCallback} [callback] Callback function.
27432767
* @returns {Promise<GetFilesResponse>}
27442768
*
@@ -2832,6 +2856,31 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
28322856
* });
28332857
* ```
28342858
*
2859+
* @example
2860+
* //-
2861+
* // Filter files using Object Contexts.
2862+
* //-
2863+
* ```
2864+
* const query = {
2865+
* filter: 'contexts."status"="active"'
2866+
* };
2867+
* bucket.getFiles(query, function(err, files) {
2868+
* if (!err) {
2869+
* // files only contains objects with the 'status' context set to 'active'.
2870+
* }
2871+
* });
2872+
*
2873+
* //-
2874+
* // You can also filter by the absence of a context key.
2875+
* //-
2876+
*
2877+
* bucket.getFiles({
2878+
* filter: '-contexts."priority":*'
2879+
* }, function(err, files) {
2880+
* // files contains objects that DO NOT have the 'priority' context key.
2881+
* });
2882+
* ```
2883+
*
28352884
* @example <caption>include:samples/files.js</caption>
28362885
* region_tag:storage_list_files
28372886
* Another example:

‎handwritten/storage/src/file.ts‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ import {
6161
unicodeJSONStringify,
6262
formatAsUTCISO,
6363
PassThroughShim,
64+
handleContextValidation,
6465
} from './util.js';
6566
import {CRC32C, CRC32CValidatorGenerator} from './crc32c.js';
6667
import {HashStreamValidator} from './hash-stream-validator.js';
@@ -382,6 +383,11 @@ export interface CopyOptions {
382383
metadata?: {
383384
[key: string]: string | boolean | number | null;
384385
};
386+
contexts?: {
387+
custom: {
388+
[key: string]: ContextValue;
389+
} | null;
390+
};
385391
predefinedAcl?: string;
386392
token?: string;
387393
userProject?: string;
@@ -485,6 +491,12 @@ export interface RestoreOptions extends PreconditionOptions {
485491
projection?: 'full' | 'noAcl';
486492
}
487493

494+
export interface ContextValue {
495+
value: string | null;
496+
readonly createTime?: string;
497+
readonly updateTime?: string;
498+
}
499+
488500
export interface FileMetadata extends BaseMetadata {
489501
acl?: AclMetadata[] | null;
490502
bucket?: string;
@@ -499,6 +511,11 @@ export interface FileMetadata extends BaseMetadata {
499511
encryptionAlgorithm?: string;
500512
keySha256?: string;
501513
};
514+
contexts?: {
515+
custom: {
516+
[key: string]: ContextValue | null;
517+
} | null;
518+
};
502519
customTime?: string;
503520
eventBasedHold?: boolean | null;
504521
readonly eventBasedHoldReleaseTime?: string;
@@ -1308,6 +1325,14 @@ class File extends ServiceObject<File, FileMetadata> {
13081325
options = {...optionsOrCallback};
13091326
}
13101327

1328+
if (options.contexts) {
1329+
const validationError = handleContextValidation(
1330+
options.contexts,
1331+
callback
1332+
);
1333+
if (validationError) return validationError;
1334+
}
1335+
13111336
callback = callback || util.noop;
13121337

13131338
let destBucket: Bucket;
@@ -4137,12 +4162,17 @@ class File extends ServiceObject<File, FileMetadata> {
41374162
optionsOrCallback?: SaveOptions | SaveCallback,
41384163
callback?: SaveCallback,
41394164
): Promise<void> | void {
4140-
// tslint:enable:no-any
41414165
callback =
41424166
typeof optionsOrCallback === 'function' ? optionsOrCallback : callback;
41434167
const options =
41444168
typeof optionsOrCallback === 'object' ? optionsOrCallback : {};
41454169

4170+
const validationError = handleContextValidation(
4171+
options.metadata?.contexts,
4172+
callback
4173+
);
4174+
if (validationError) return validationError;
4175+
41464176
let maxRetries = this.storage.retryOptions.maxRetries;
41474177
if (
41484178
!this.shouldRetryBasedOnPreconditionAndIdempotencyStrat(
@@ -4246,6 +4276,9 @@ class File extends ServiceObject<File, FileMetadata> {
42464276
? (optionsOrCallback as MetadataCallback<FileMetadata>)
42474277
: cb;
42484278

4279+
const validationError = handleContextValidation(metadata.contexts, cb);
4280+
if (validationError) return validationError;
4281+
42494282
this.disableAutoRetryConditionallyIdempotent_(
42504283
this.methods.setMetadata,
42514284
AvailableServiceObjectMethods.setMetadata,

‎handwritten/storage/src/util.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import * as url from 'url';
1919
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
2020
// @ts-ignore
2121
import {getPackageJSON} from './package-json-helper.cjs';
22+
import {FileMetadata} from './file';
2223

2324
// Done to avoid a problem with mangling of identifiers when using esModuleInterop
2425
const fileURLToPath = url.fileURLToPath;
@@ -272,3 +273,49 @@ export class PassThroughShim extends PassThrough {
272273
callback(null);
273274
}
274275
}
276+
277+
278+
/**
279+
* Validates Object Contexts for forbidden characters.
280+
* Double quotes (") are forbidden in context keys and values as they
281+
* interfere with GCS filter string syntax.
282+
*
283+
* @param {FileMetadata['contexts']} contexts The contexts object to validate.
284+
* @returns {void} Throws an error if validation fails.
285+
*/
286+
export function validateContexts(contexts?: FileMetadata['contexts']): void {
287+
const custom = contexts?.custom;
288+
if (!custom) return;
289+
for (const [key, context] of Object.entries(custom)) {
290+
if (key.includes('"')) {
291+
throw new Error(
292+
`Invalid context key "${key}": Forbidden character (") detected.`
293+
);
294+
}
295+
if (context?.value && context.value.includes('"')) {
296+
throw new Error(
297+
`Invalid context value for key "${key}": Forbidden character (") detected.`
298+
);
299+
}
300+
}
301+
}
302+
303+
/**
304+
* Helper to validate contexts and route errors to either a callback or a Promise.
305+
* @param contexts The contexts to validate.
306+
* @param callback The optional user-provided callback.
307+
*/
308+
export function handleContextValidation(
309+
contexts?: FileMetadata['contexts'],
310+
callback?: Function
311+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
312+
): Promise<any> | void {
313+
try {
314+
validateContexts(contexts);
315+
} catch (err) {
316+
if (callback) {
317+
return callback(err as Error);
318+
}
319+
return Promise.reject(err);
320+
}
321+
}

0 commit comments

Comments
 (0)