diff --git a/handwritten/storage/conformance-test/conformanceCommon.ts b/handwritten/storage/conformance-test/conformanceCommon.ts index 3ffd0faa6daf..cb7f1016f550 100644 --- a/handwritten/storage/conformance-test/conformanceCommon.ts +++ b/handwritten/storage/conformance-test/conformanceCommon.ts @@ -16,21 +16,17 @@ // eslint-disable-next-line @typescript-eslint/no-unused-vars import * as jsonToNodeApiMapping from './test-data/retryInvocationMap.json'; import * as libraryMethods from './libraryMethods'; -import { - Bucket, - File, - GaxiosOptions, - GaxiosOptionsPrepared, - HmacKey, - Notification, - Storage, -} from '../src'; +import {Bucket, File, Gaxios, HmacKey, Notification, Storage} from '../src'; import * as uuid from 'uuid'; import * as assert from 'assert'; import { StorageRequestOptions, StorageTransport, + StorageTransportCallback, } from '../src/storage-transport'; +import {getDirName} from '../src/util'; +import path from 'path'; +import {GoogleAuth} from 'google-auth-library'; interface RetryCase { instructions: String[]; } @@ -60,7 +56,7 @@ interface ConformanceTestResult { type LibraryMethodsModuleType = typeof import('./libraryMethods'); const methodMap: Map = new Map( - Object.entries({}), // TODO: replace with Object.entries(jsonToNodeApiMapping) + Object.entries(jsonToNodeApiMapping), ); const DURATION_SECONDS = 600; // 10 mins. @@ -70,6 +66,21 @@ const TESTBENCH_HOST = const CONF_TEST_PROJECT_ID = 'my-project-id'; const TIMEOUT_FOR_INDIVIDUAL_TEST = 20000; const RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS = 0.01; +const SERVICE_ACCOUNT = path.join( + getDirName(), + '../../../conformance-test/fixtures/signing-service-account.json', +); + +const authClient = new GoogleAuth({ + keyFilename: SERVICE_ACCOUNT, + scopes: ['https://www.googleapis.com/auth/devstorage.full_control'], +}).fromJSON(require(SERVICE_ACCOUNT)); + +authClient.getAccessToken = async () => ({token: 'unauthenticated-test-token'}); +authClient.request = async opts => { + const gaxios = new Gaxios(); + return gaxios.request(opts); +}; export function executeScenario(testCase: RetryTestCase) { for ( @@ -89,16 +100,17 @@ export function executeScenario(testCase: RetryTestCase) { let bucket: Bucket; let file: File; let notification: Notification; - let creationResult: {id: string}; + let creationResult: ConformanceTestCreationResult; let storage: Storage; let hmacKey: HmacKey; let storageTransport: StorageTransport; describe(`${storageMethodString}`, async () => { beforeEach(async () => { - storageTransport = new StorageTransport({ + const rawTransport = new StorageTransport({ apiEndpoint: TESTBENCH_HOST, - authClient: undefined, + authClient: authClient, + keyFilename: SERVICE_ACCOUNT, baseUrl: TESTBENCH_HOST, packageJson: {name: 'test-package', version: '1.0.0'}, retryOptions: { @@ -117,92 +129,71 @@ export function executeScenario(testCase: RetryTestCase) { timeout: DURATION_SECONDS, }); + creationResult = await createTestBenchRetryTest( + instructionSet.instructions, + jsonMethod?.name.toString(), + rawTransport, + ); + + // Create a Proxy around rawStorageTransport to intercept makeRequest + storageTransport = createRetryProxy( + rawTransport, + creationResult.id, + ); + storage = new Storage({ apiEndpoint: TESTBENCH_HOST, projectId: CONF_TEST_PROJECT_ID, + keyFilename: SERVICE_ACCOUNT, + authClient: authClient, retryOptions: { retryDelayMultiplier: RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS, }, }); - creationResult = await createTestBenchRetryTest( - instructionSet.instructions, - jsonMethod?.name.toString(), - storageTransport, + bucket = await createBucketForTest( + storage, + testCase.preconditionProvided && + !storageMethodString.includes('combine'), + storageMethodString, + ); + file = await createFileForTest( + testCase.preconditionProvided, + storageMethodString, + bucket, ); - if (storageMethodString.includes('InstancePrecondition')) { - bucket = await createBucketForTest( - storage, - testCase.preconditionProvided, - storageMethodString, - ); - file = await createFileForTest( - testCase.preconditionProvided, - storageMethodString, - bucket, - ); - } else { - bucket = await createBucketForTest( - storage, - false, - storageMethodString, - ); - file = await createFileForTest( - false, - storageMethodString, - bucket, - ); - } notification = bucket.notification(TESTS_PREFIX); await notification.create(); [hmacKey] = await storage.createHmacKey( `${TESTS_PREFIX}@email.com`, ); - - storage.interceptors.push({ - resolved: ( - requestConfig: GaxiosOptionsPrepared, - ): Promise => { - const config = requestConfig as GaxiosOptions; - config.headers = config.headers || {}; - Object.assign(config.headers, { - 'x-retry-test-id': creationResult.id, - }); - return Promise.resolve(config as GaxiosOptionsPrepared); - }, - rejected: error => { - return Promise.reject(error); - }, - }); }); it(`${instructionNumber}`, async () => { const methodParameters: libraryMethods.ConformanceTestOptions = { - storage: storage, - bucket: bucket, - file: file, - storageTransport: storageTransport, - notification: notification, - hmacKey: hmacKey, + storage, + bucket, + file, + storageTransport, + notification, + hmacKey, + projectId: CONF_TEST_PROJECT_ID, + preconditionRequired: testCase.preconditionProvided, }; - if (testCase.preconditionProvided) { - methodParameters.preconditionRequired = true; - } if (testCase.expectSuccess) { - assert.ifError(await storageMethodObject(methodParameters)); + await storageMethodObject(methodParameters); + const testBenchResult = await getTestBenchRetryTest( + creationResult.id, + storageTransport, + ); + assert.strictEqual(testBenchResult.completed, true); } else { await assert.rejects(async () => { await storageMethodObject(methodParameters); }, undefined); } - - const testBenchResult = await getTestBenchRetryTest( - creationResult.id, - storageTransport, - ); - assert.strictEqual(testBenchResult.completed, true); }).timeout(TIMEOUT_FOR_INDIVIDUAL_TEST); }); }); @@ -210,20 +201,51 @@ export function executeScenario(testCase: RetryTestCase) { } } +/** + * Creates a Proxy to automatically inject x-retry-test-id into all requests + */ +function createRetryProxy( + transport: StorageTransport, + retryId: string, +): StorageTransport { + return new Proxy(transport, { + get(target, prop, receiver) { + const original = Reflect.get(target, prop, receiver); + if (prop === 'makeRequest' && typeof original === 'function') { + return async ( + reqOpts: StorageRequestOptions, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + callback?: StorageTransportCallback, + ) => { + reqOpts.headers = reqOpts.headers || {}; + + if (reqOpts.headers instanceof Headers) { + reqOpts.headers.set('x-retry-test-id', retryId); + } else { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (reqOpts.headers as any)['x-retry-test-id'] = retryId; + } + + return original.apply(target, [reqOpts, callback]); + }; + } + return original; + }, + }); +} + async function createBucketForTest( storage: Storage, - preconditionShouldBeOnInstance: boolean, - storageMethodString: String, + withPrecondition: boolean, + method: String, ) { - const name = generateName(storageMethodString, 'bucket'); - const bucket = storage.bucket(name); - await bucket.create(); + const bucket = storage.bucket(generateName(method, 'bucket')); + const [metadata] = await bucket.create(); await bucket.setRetentionPeriod(DURATION_SECONDS); - - if (preconditionShouldBeOnInstance) { + if (withPrecondition) { return new Bucket(storage, bucket.name, { preconditionOpts: { - ifMetagenerationMatch: 2, + ifMetagenerationMatch: metadata.metageneration, }, }); } @@ -231,61 +253,52 @@ async function createBucketForTest( } async function createFileForTest( - preconditionShouldBeOnInstance: boolean, - storageMethodString: String, + withPrecondition: boolean, + method: String, bucket: Bucket, ) { - const name = generateName(storageMethodString, 'file'); - const file = bucket.file(name); - await file.save(name); - if (preconditionShouldBeOnInstance) { + const file = bucket.file(generateName(method, 'file')); + await file.save('test-content'); + if (withPrecondition) { + const [metadata] = await file.getMetadata(); return new File(bucket, file.name, { preconditionOpts: { - ifMetagenerationMatch: file.metadata.metageneration, - ifGenerationMatch: file.metadata.generation, + ifMetagenerationMatch: metadata.metageneration, + ifGenerationMatch: metadata.generation, }, }); } return file; } -function generateName(storageMethodString: String, bucketOrFile: string) { - return `${TESTS_PREFIX}${storageMethodString.toLowerCase()}${bucketOrFile}.${shortUUID()}`; -} - async function createTestBenchRetryTest( instructions: String[], methodName: string, - storageTransport: StorageTransport, + transport: StorageTransport, ): Promise { - const requestBody = {instructions: {[methodName]: instructions}}; - - const requestOptions: StorageRequestOptions = { + return (await transport.makeRequest({ method: 'POST', url: 'retry_test', - body: JSON.stringify(requestBody), + body: JSON.stringify({instructions: {[methodName]: instructions}}), headers: {'Content-Type': 'application/json'}, - }; - - const response = await storageTransport.makeRequest(requestOptions); - return response as unknown as ConformanceTestCreationResult; + })) as ConformanceTestCreationResult; } async function getTestBenchRetryTest( testId: string, - storageTransport: StorageTransport, + transport: StorageTransport, ): Promise { - const response = await storageTransport.makeRequest({ + return (await transport.makeRequest({ url: `retry_test/${testId}`, method: 'GET', - retry: true, - headers: { - 'x-retry-test-id': testId, - }, - }); - return response as unknown as ConformanceTestResult; + headers: {'x-retry-test-id': testId}, + })) as ConformanceTestResult; +} + +function generateName(method: String, type: string) { + return `${TESTS_PREFIX}${method.toLowerCase()}${type}.${shortUUID()}`; } function shortUUID() { - return uuid.v1().split('-').shift(); + return uuid.v4().split('-').shift(); } diff --git a/handwritten/storage/conformance-test/libraryMethods.ts b/handwritten/storage/conformance-test/libraryMethods.ts index 26c466143b85..81f330829dee 100644 --- a/handwritten/storage/conformance-test/libraryMethods.ts +++ b/handwritten/storage/conformance-test/libraryMethods.ts @@ -21,18 +21,14 @@ import { Policy, GaxiosError, } from '../src'; -import * as path from 'path'; -import { - createTestBuffer, - createTestFileFromBuffer, - deleteTestFile, -} from './testBenchUtil'; +import {createTestBuffer} from './testBenchUtil'; import * as uuid from 'uuid'; -import {getDirName} from '../src/util.js'; -import {StorageTransport} from '../src/storage-transport'; +import { + StorageTransport, + StorageRequestOptions, +} from '../src/storage-transport'; const FILE_SIZE_BYTES = 9 * 1024 * 1024; -const CHUNK_SIZE_BYTES = 2 * 1024 * 1024; export interface ConformanceTestOptions { bucket?: Bucket; @@ -42,6 +38,8 @@ export interface ConformanceTestOptions { hmacKey?: HmacKey; preconditionRequired?: boolean; storageTransport?: StorageTransport; + projectId?: string; + retryTestId?: string; } ///////////////////////////////////////////////// @@ -51,63 +49,37 @@ export interface ConformanceTestOptions { export async function addLifecycleRuleInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.addLifecycleRule({ - action: { - type: 'Delete', - }, - condition: { - age: 365 * 3, // Specified in days. - }, - }); + return await addLifecycleRule(options); } export async function addLifecycleRule(options: ConformanceTestOptions) { - if (options.preconditionRequired) { - await options.bucket!.addLifecycleRule( - { - action: { - type: 'Delete', - }, - condition: { - age: 365 * 3, // Specified in days. - }, - }, - { - ifMetagenerationMatch: 2, - }, - ); - } else { - await options.bucket!.addLifecycleRule({ - action: { - type: 'Delete', - }, - condition: { - age: 365 * 3, // Specified in days. + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + lifecycle: { + rule: [ + { + action: {type: 'Delete'}, + condition: {age: 1095}, // Specified in days. + }, + ], }, - }); + }), + queryParameters: {}, + }; + + if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function combineInstancePrecondition( options: ConformanceTestOptions, ) { - const file1 = options.bucket!.file('file1.txt'); - const file2 = options.bucket!.file('file2.txt'); - await file1.save('file1 contents'); - await file2.save('file2 contents'); - let allFiles; - const sources = [file1, file2]; - if (options.preconditionRequired) { - allFiles = options.bucket!.file('all-files.txt', { - preconditionOpts: { - ifGenerationMatch: 0, - }, - }); - } else { - allFiles = options.bucket!.file('all-files.txt'); - } - - await options.bucket!.combine(sources, allFiles); + return await combine(options); } export async function combine(options: ConformanceTestOptions) { @@ -115,36 +87,122 @@ export async function combine(options: ConformanceTestOptions) { const file2 = options.bucket!.file('file2.txt'); await file1.save('file1 contents'); await file2.save('file2 contents'); - const sources = [file1, file2]; - const allFiles = options.bucket!.file('all-files.txt'); - await allFiles.save('allfiles contents'); + + const destinationFile = encodeURIComponent('all-files.txt'); + const body = { + sourceObjects: [{name: file1.name}, {name: file2.name}], + }; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${destinationFile}/compose`, + body: JSON.stringify(body), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.combine(sources, allFiles, { - ifGenerationMatch: allFiles.metadata.generation!, - }); + requestOptions.queryParameters!.ifGenerationMatch = 0; } else { - await options.bucket!.combine(sources, allFiles); + delete requestOptions.queryParameters!.ifGenerationMatch; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function create(options: ConformanceTestOptions) { - const [bucketExists] = await options.bucket!.exists(); + if (!options.storageTransport || !options.projectId || !options.bucket) { + throw new Error( + 'storageTransport, projectId, and bucket are required for the create test.', + ); + } + const bucketName = options.bucket.name; + let bucketExists = false; + const existsReq: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${bucketName}`, + }; + try { + await options.storageTransport.makeRequest(existsReq); + bucketExists = true; + } catch (e) { + const err = e as GaxiosError; + if (err.response?.status !== 404) { + throw e; + } + } + if (bucketExists) { - await options.bucket!.deleteFiles(); - await options.bucket!.delete({ - ignoreNotFound: true, - }); + let pageToken: string | undefined = undefined; + do { + const listReq: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${bucketName}/o`, + queryParameters: pageToken ? {pageToken} : undefined, + }; + const listResult = await options.storageTransport.makeRequest(listReq); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const objects = (listResult as any)?.items || []; + + for (const obj of objects) { + const deleteObjReq: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${bucketName}/o/${obj.name}`, + }; + await options.storageTransport.makeRequest(deleteObjReq); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + pageToken = (listResult as any)?.nextPageToken; + } while (pageToken); + + const deleteBucketReq: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${bucketName}`, + }; + await options.storageTransport.makeRequest(deleteBucketReq); } - await options.bucket!.create(); + + const createRequest: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b?project=${options.projectId}`, + body: JSON.stringify({name: bucketName}), + headers: {'Content-Type': 'application/json'}, + }; + await options.storageTransport.makeRequest(createRequest); } export async function createNotification(options: ConformanceTestOptions) { - await options.bucket!.createNotification('my-topic'); + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs`, + body: JSON.stringify({ + topic: 'my-topic', + }), + headers: {'Content-Type': 'application/json'}, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function deleteBucket(options: ConformanceTestOptions) { - await options.bucket!.deleteFiles(); - await options.bucket!.delete(); + try { + await options.bucket!.deleteFiles(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + const message = err.message || ''; + if (!message.includes('does not exist') && err.code !== 404) { + throw err; + } + } + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + queryParameters: {}, + }; + + if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = 1; + } + return await options.storageTransport!.makeRequest(requestOptions); } // Note: bucket.deleteFiles is missing from these tests @@ -153,331 +211,455 @@ export async function deleteBucket(options: ConformanceTestOptions) { export async function deleteLabelsInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.deleteLabels(); + return await deleteLabels(options); } export async function deleteLabels(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({labels: null}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.deleteLabels({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.deleteLabels(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function disableRequesterPaysInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.disableRequesterPays(); + return await disableRequesterPays(options); } export async function disableRequesterPays(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({billing: {requesterPays: false}}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.disableRequesterPays({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.disableRequesterPays(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function enableLoggingInstancePrecondition( options: ConformanceTestOptions, ) { - const config = { - prefix: 'log', - }; - await options.bucket!.enableLogging(config); + return await enableLogging(options); } export async function enableLogging(options: ConformanceTestOptions) { - let config; + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + logging: { + logBucket: options.bucket!.name, + logObjectPrefix: 'log', + }, + }), + queryParameters: {}, + }; + if (options.preconditionRequired) { - config = { - prefix: 'log', - ifMetagenerationMatch: 2, - }; - } else { - config = { - prefix: 'log', - }; + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } - await options.bucket!.enableLogging(config); + + return await options.storageTransport!.makeRequest(requestOptions); } export async function enableRequesterPaysInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.enableRequesterPays(); + return await enableRequesterPays(options); } export async function enableRequesterPays(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({billing: {requesterPays: true}}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.enableRequesterPays({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.enableRequesterPays(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketExists(options: ConformanceTestOptions) { - await options.bucket!.exists(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + }; + + try { + await options.storageTransport!.makeRequest(requestOptions); + return true; + } catch (err: unknown) { + const gaxiosError = err as GaxiosError; + if (gaxiosError.response?.status === 404) { + return false; + } + throw err; + } } export async function bucketGet(options: ConformanceTestOptions) { - await options.bucket!.get(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getFilesStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .bucket!.getFilesStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', (err: GaxiosError) => reject(err)); - }); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getLabels(options: ConformanceTestOptions) { - await options.bucket!.getLabels(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + queryParameters: { + fields: 'labels', + }, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketGetMetadata(options: ConformanceTestOptions) { - await options.bucket!.getMetadata(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + queryParameters: { + projection: 'full', + }, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getNotifications(options: ConformanceTestOptions) { - await options.bucket!.getNotifications(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function lock(options: ConformanceTestOptions) { - const metageneration = 0; - await options.bucket!.lock(metageneration); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const metadata: any = await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + headers: {...(retryId ? {'x-retry-test-id': retryId} : {})}, + }); + const currentMetageneration = metadata.metageneration; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/lockRetentionPolicy`, + queryParameters: { + ifMetagenerationMatch: currentMetageneration, + }, + headers: { + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketMakePrivateInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.makePrivate(); + return await bucketMakePrivate(options); } export async function bucketMakePrivate(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({acl: []}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.makePrivate({ - preconditionOpts: {ifMetagenerationMatch: 2}, - }); - } else { - await options.bucket!.makePrivate(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketMakePublic(options: ConformanceTestOptions) { - await options.bucket!.makePublic(); + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function removeRetentionPeriodInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.removeRetentionPeriod(); + return await removeRetentionPeriod(options); } export async function removeRetentionPeriod(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({retentionPolicy: null}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.removeRetentionPeriod({ - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.removeRetentionPeriod(); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setCorsConfigurationInstancePrecondition( options: ConformanceTestOptions, ) { - const corsConfiguration = [{maxAgeSeconds: 3600}]; // 1 hour - await options.bucket!.setCorsConfiguration(corsConfiguration); + return await setCorsConfiguration(options); } export async function setCorsConfiguration(options: ConformanceTestOptions) { - const corsConfiguration = [{maxAgeSeconds: 3600}]; // 1 hour + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({cors: [{maxAgeSeconds: 3600}]}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.setCorsConfiguration(corsConfiguration, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setCorsConfiguration(corsConfiguration); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setLabelsInstancePrecondition( options: ConformanceTestOptions, ) { - const labels = { - labelone: 'labelonevalue', - labeltwo: 'labeltwovalue', - }; - await options.bucket!.setLabels(labels); + return await setLabels(options); } export async function setLabels(options: ConformanceTestOptions) { - const labels = { - labelone: 'labelonevalue', - labeltwo: 'labeltwovalue', + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + labels: {labelone: 'labelonevalue', labeltwo: 'labeltwovalue'}, + }), + queryParameters: {}, }; + if (options.preconditionRequired) { - await options.bucket!.setLabels(labels, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setLabels(labels); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketSetMetadataInstancePrecondition( options: ConformanceTestOptions, ) { - const metadata = { - website: { - mainPageSuffix: 'http://example.com', - notFoundPage: 'http://example.com/404.html', - }, - }; - await options.bucket!.setMetadata(metadata); + return await bucketSetMetadata(options); } export async function bucketSetMetadata(options: ConformanceTestOptions) { - const metadata = { - website: { - mainPageSuffix: 'http://example.com', - notFoundPage: 'http://example.com/404.html', - }, + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + website: { + mainPageSuffix: 'http://example.com', + notFoundPage: 'http://example.com/404.html', + }, + }), + queryParameters: {}, }; + if (options.preconditionRequired) { - await options.bucket!.setMetadata(metadata, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setMetadata(metadata); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setRetentionPeriodInstancePrecondition( options: ConformanceTestOptions, ) { - const DURATION_SECONDS = 15780000; // 6 months. - await options.bucket!.setRetentionPeriod(DURATION_SECONDS); + return await setRetentionPeriod(options); } export async function setRetentionPeriod(options: ConformanceTestOptions) { - const DURATION_SECONDS = 15780000; // 6 months. + const DURATION_SECONDS = 15780000; + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({ + retentionPolicy: {retentionPeriod: DURATION_SECONDS.toString()}, + }), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.setRetentionPeriod(DURATION_SECONDS, { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setRetentionPeriod(DURATION_SECONDS); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketSetStorageClassInstancePrecondition( options: ConformanceTestOptions, ) { - await options.bucket!.setStorageClass('nearline'); + return await bucketSetStorageClass(options); } export async function bucketSetStorageClass(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}`, + body: JSON.stringify({storageClass: 'NEARLINE'}), + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.bucket!.setStorageClass('nearline', { - ifMetagenerationMatch: 2, - }); - } else { - await options.bucket!.setStorageClass('nearline'); + requestOptions.queryParameters!.ifMetagenerationMatch = 2; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function bucketUploadResumableInstancePrecondition( options: ConformanceTestOptions, ) { - const filePath = path.join( - getDirName(), - `../conformance-test/test-data/tmp-${uuid.v4()}.txt`, - ); - createTestFileFromBuffer(FILE_SIZE_BYTES, filePath); - if (options.bucket!.instancePreconditionOpts) { - options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0; - delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; - } - await options.bucket!.upload(filePath, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); - deleteTestFile(filePath); + return await bucketUploadResumable(options); } export async function bucketUploadResumable(options: ConformanceTestOptions) { - const filePath = path.join( - getDirName(), - `../conformance-test/test-data/tmp-${uuid.v4()}.txt`, - ); - createTestFileFromBuffer(FILE_SIZE_BYTES, filePath); + const fileName = `resumable-file-${uuid.v4()}.txt`; + const dataBuffer = Buffer.alloc(FILE_SIZE_BYTES, 'a'); + + const initiateOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'resumable', + name: fileName, + }, + headers: { + 'X-Upload-Content-Type': 'text/plain', + 'X-Upload-Content-Length': FILE_SIZE_BYTES.toString(), + }, + body: JSON.stringify({name: fileName}), + }; + if (options.preconditionRequired) { - await options.bucket!.upload(filePath, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.bucket!.upload(filePath, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); + initiateOptions.queryParameters = initiateOptions.queryParameters || {}; + initiateOptions.queryParameters.ifGenerationMatch = 0; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const response: any = + await options.storageTransport!.makeRequest(initiateOptions); + + let sessionUri: string | null = null; + + if (response.headers) { + if (typeof response.headers.get === 'function') { + sessionUri = response.headers.get('location'); + } + if (!sessionUri) { + sessionUri = response.headers.location || response.headers.Location; + } + } + + if (!sessionUri && response.data && typeof response.data === 'object') { + sessionUri = response.data.location; + } + + if (!sessionUri) { + throw new Error( + 'Failed to get session URI from resumable upload initiation.', + ); } - deleteTestFile(filePath); + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: sessionUri, + body: dataBuffer, + queryParameters: undefined, + headers: { + 'Content-Length': FILE_SIZE_BYTES.toString(), + 'Content-Range': `bytes 0-${FILE_SIZE_BYTES - 1}/${FILE_SIZE_BYTES}`, + }, + }); } export async function bucketUploadMultipartInstancePrecondition( options: ConformanceTestOptions, ) { - if (options.bucket!.instancePreconditionOpts) { - delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; - options.bucket!.instancePreconditionOpts.ifGenerationMatch = 0; - } - await options.bucket!.upload( - path.join( - getDirName(), - '../../../conformance-test/test-data/retryStrategyTestData.json', - ), - {resumable: false}, - ); + return await bucketUploadMultipart(options); } export async function bucketUploadMultipart(options: ConformanceTestOptions) { - if (options.bucket!.instancePreconditionOpts) { - delete options.bucket!.instancePreconditionOpts.ifMetagenerationMatch; - } + const fileName = 'retryStrategyTestData.json'; + const boundary = 'foo_bar_baz'; + + const metadata = JSON.stringify({ + name: fileName, + contentType: 'application/json', + }); + const media = JSON.stringify({some: 'data'}); + const body = + `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${metadata}\r\n` + + `--${boundary}\r\nContent-Type: application/json\r\n\r\n${media}\r\n` + + `--${boundary}--`; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'multipart', + name: fileName, + }, + headers: {'Content-Type': `multipart/related; boundary=${boundary}`}, + body: body, + }; if (options.preconditionRequired) { - await options.bucket!.upload( - path.join( - getDirName(), - '../../../conformance-test/test-data/retryStrategyTestData.json', - ), - {resumable: false, preconditionOpts: {ifGenerationMatch: 0}}, - ); - } else { - await options.bucket!.upload( - path.join( - getDirName(), - '../../../conformance-test/test-data/retryStrategyTestData.json', - ), - {resumable: false}, - ); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } ///////////////////////////////////////////////// @@ -485,195 +667,380 @@ export async function bucketUploadMultipart(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function copy(options: ConformanceTestOptions) { - const newFile = new File(options.bucket!, 'a-different-file.png'); - await newFile.save('a-different-file.png'); + const sourceBucket = options.bucket!.name; + const sourceFile = encodeURIComponent(options.file!.name); + const destinationFile = encodeURIComponent('a-different-file.png'); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${sourceBucket}/o/${sourceFile}/rewriteTo/b/${sourceBucket}/o/${destinationFile}`, + queryParameters: {}, + }; if (options.preconditionRequired) { - await options.file!.copy('a-different-file.png', { - preconditionOpts: { - ifGenerationMatch: newFile.metadata.generation!, - }, - }); - } else { - await options.file!.copy('a-different-file.png'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifSourceGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function createReadStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .file!.createReadStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', (err: GaxiosError) => reject(err)); - }); + return await download(options); } export async function createResumableUploadInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.createResumableUpload(); + return await createResumableUpload(options); } export async function createResumableUpload(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'resumable', + name: options.file!.name, + }, + }; + if (options.preconditionRequired) { - await options.file!.createResumableUpload({ - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.file!.createResumableUpload(); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileDeleteInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.delete(); + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + headers: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + 'x-retry-test-id': (options as any).retryTestId, + }, + }; + if (options.preconditionRequired) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifGenerationMatch = generation; + } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileDelete(options: ConformanceTestOptions) { + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + headers: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + 'x-retry-test-id': (options as any).retryTestId, + }, + }; + if (options.preconditionRequired) { - await options.file!.delete({ - ifGenerationMatch: options.file!.metadata.generation, - }); - } else { - await options.file!.delete(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function download(options: ConformanceTestOptions) { - await options.file!.download(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {alt: 'media'}, + responseType: 'stream', + headers: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ...(options as any).headers, + }, + }; + return await options.storageTransport!.makeRequest(requestOptions); } export async function exists(options: ConformanceTestOptions) { - await options.file!.exists(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + }; + + try { + await options.storageTransport!.makeRequest(requestOptions); + return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if (err.code === 404) return false; + throw err; + } } export async function get(options: ConformanceTestOptions) { - await options.file!.get(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function getExpirationDate(options: ConformanceTestOptions) { - await options.file!.getExpirationDate(); + return await get(options); } export async function getMetadata(options: ConformanceTestOptions) { - await options.file!.getMetadata(); + return await get(options); } export async function isPublic(options: ConformanceTestOptions) { - await options.file!.isPublic(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${options.bucket!.name}/o/${encodeURIComponent(options.file!.name)}`, + }; + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileMakePrivateInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.makePrivate(); + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + body: JSON.stringify({acl: []}), + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + if (instanceOpts?.ifGenerationMatch !== undefined) { + requestOptions.queryParameters!.ifGenerationMatch = + instanceOpts.ifGenerationMatch; + } else if (instanceOpts?.ifMetagenerationMatch !== undefined) { + requestOptions.queryParameters!.ifMetagenerationMatch = + instanceOpts.ifMetagenerationMatch; + } else if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = + options.file?.metadata.metageneration; + } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileMakePrivate(options: ConformanceTestOptions) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + body: JSON.stringify({acl: []}), + queryParameters: {}, + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }; + if (options.preconditionRequired) { - await options.file!.makePrivate({ - preconditionOpts: { - ifMetagenerationMatch: options.file!.metadata.metageneration, - }, - }); - } else { - await options.file!.makePrivate(); + requestOptions.queryParameters!.ifMetagenerationMatch = + options.file!.metadata.metageneration ?? 1; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function fileMakePublic(options: ConformanceTestOptions) { - await options.file!.makePublic(); + const fileName = encodeURIComponent(options.file!.name); + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${fileName}/acl`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function move(options: ConformanceTestOptions) { + const sourceBucket = options.bucket!.name; + const sourceFile = encodeURIComponent(options.file!.name); + const destinationFile = encodeURIComponent('new-file'); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${sourceBucket}/o/${sourceFile}/rewriteTo/b/${sourceBucket}/o/${destinationFile}`, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.move('new-file', { - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.file!.move('new-file'); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function rename(options: ConformanceTestOptions) { + const sourceBucket = options.bucket!.name; + const sourceFile = encodeURIComponent(options.file!.name); + const destinationFile = encodeURIComponent('new-name'); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${sourceBucket}/o/${sourceFile}/rewriteTo/b/${sourceBucket}/o/${destinationFile}`, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.rename('new-name', { - preconditionOpts: {ifGenerationMatch: 0}, - }); - } else { - await options.file!.rename('new-name'); + requestOptions.queryParameters!.ifGenerationMatch = 0; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function rotateEncryptionKey(options: ConformanceTestOptions) { - const crypto = require('crypto'); - const buffer = crypto.randomBytes(32); - const newKey = buffer.toString('base64'); + const bucketName = options.bucket!.name; + const fileName = encodeURIComponent(options.file!.name); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${bucketName}/o/${fileName}/rewriteTo/b/${bucketName}/o/${fileName}`, + headers: { + 'x-goog-copy-source-encryption-algorithm': 'AES256', + }, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.rotateEncryptionKey({ - encryptionKey: Buffer.from(newKey, 'base64'), - preconditionOpts: {ifGenerationMatch: options.file!.metadata.generation}, - }); - } else { - await options.file!.rotateEncryptionKey({ - encryptionKey: Buffer.from(newKey, 'base64'), - }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function saveResumableInstancePrecondition( options: ConformanceTestOptions, ) { - const buf = createTestBuffer(FILE_SIZE_BYTES); - await options.file!.save(buf, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); + return await saveResumable(options); } export async function saveResumable(options: ConformanceTestOptions) { - const buf = createTestBuffer(FILE_SIZE_BYTES); + const data = createTestBuffer(FILE_SIZE_BYTES); + const dataBuffer = Buffer.from(data); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + + const initiateOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'resumable', + name: options.file!.name, + }, + body: JSON.stringify({name: options.file!.name}), + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }; + if (options.preconditionRequired) { - await options.file!.save(buf, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - preconditionOpts: { - ifGenerationMatch: options.file!.metadata.generation, - ifMetagenerationMatch: options.file!.metadata.metageneration, - }, - }); - } else { - await options.file!.save(buf, { - resumable: true, - chunkSize: CHUNK_SIZE_BYTES, - metadata: {contentLength: FILE_SIZE_BYTES}, - }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + initiateOptions.queryParameters!.ifGenerationMatch = generation; + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const response: any = + await options.storageTransport!.makeRequest(initiateOptions); + let sessionUri: string | null = null; + if (response.headers) { + if (typeof response.headers.get === 'function') { + sessionUri = response.headers.get('location'); + } + if (!sessionUri) { + sessionUri = response.headers.location || response.headers.Location; + } } + + if (!sessionUri && response.data && typeof response.data === 'object') { + sessionUri = response.data.location; + } + + if (!sessionUri) throw new Error('Failed to get session URI'); + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: sessionUri, + body: dataBuffer, + queryParameters: undefined, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': dataBuffer.length.toString(), + 'Content-Range': `bytes 0-${dataBuffer.length - 1}/${dataBuffer.length}`, + 'x-retry-test-id': retryId, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any, + }); } export async function saveMultipartInstancePrecondition( options: ConformanceTestOptions, ) { - await options.file!.save('testdata', {resumable: false}); + return await saveMultipart(options); } export async function saveMultipart(options: ConformanceTestOptions) { - if (options.preconditionRequired) { - await options.file!.save('testdata', { - resumable: false, - preconditionOpts: { - ifGenerationMatch: options.file!.metadata.generation, - }, - }); - } else { - await options.file!.save('testdata', { - resumable: false, - }); + const boundary = 'conformance_test_boundary'; + const fileName = options.file!.name; + + const metadata = JSON.stringify({name: fileName}); + const media = 'testdata'; + + const body = + `--${boundary}\r\nContent-Type: application/json; charset=UTF-8\r\n\r\n${metadata}\r\n` + + `--${boundary}\r\nContent-Type: text/plain\r\n\r\n${media}\r\n` + + `--${boundary}--`; + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `upload/storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o`, + queryParameters: { + uploadType: 'multipart', + name: options.file!.name, + }, + headers: {'Content-Type': `multipart/related; boundary=${boundary}`}, + body: body, + }; + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + if (instanceOpts?.ifGenerationMatch !== undefined) { + requestOptions.queryParameters!.ifGenerationMatch = + instanceOpts.ifGenerationMatch; + } else if (options.preconditionRequired) { + const generation = options.file?.metadata?.generation ?? 0; + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setMetadataInstancePrecondition( @@ -681,41 +1048,79 @@ export async function setMetadataInstancePrecondition( ) { const metadata = { contentType: 'application/x-font-ttf', - metadata: { - my: 'custom', - properties: 'go here', - }, + metadata: {my: 'custom', properties: 'go here'}, + }; + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/o/${encodeURIComponent(options.file!.name)}`, + queryParameters: {}, + body: JSON.stringify(metadata), + headers: {'Content-Type': 'application/json'}, }; - await options.file!.setMetadata(metadata); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + + if (instanceOpts?.ifGenerationMatch !== undefined) { + requestOptions.queryParameters!.ifGenerationMatch = + instanceOpts.ifGenerationMatch; + } else if (instanceOpts?.ifMetagenerationMatch !== undefined) { + requestOptions.queryParameters!.ifMetagenerationMatch = + instanceOpts.ifMetagenerationMatch; + } else if (options.preconditionRequired) { + requestOptions.queryParameters!.ifMetagenerationMatch = + options.file?.metadata.metageneration; + } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setMetadata(options: ConformanceTestOptions) { const metadata = { contentType: 'application/x-font-ttf', - metadata: { - my: 'custom', - properties: 'go here', - }, + metadata: {my: 'custom', properties: 'go here'}, }; + + const requestOptions: StorageRequestOptions = { + method: 'PATCH', + url: `storage/v1/b/${options.bucket!.name}/o/${encodeURIComponent(options.file!.name)}`, + body: JSON.stringify(metadata), + headers: {'Content-Type': 'application/json'}, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.setMetadata(metadata, { - ifMetagenerationMatch: options.file!.metadata.metageneration, - }); - } else { - await options.file!.setMetadata(metadata); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + + requestOptions.queryParameters!.ifGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setStorageClass(options: ConformanceTestOptions) { + const bucketName = options.bucket!.name; + const fileName = encodeURIComponent(options.file!.name); + + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${bucketName}/o/${fileName}/rewriteTo/b/${bucketName}/o/${fileName}`, + body: JSON.stringify({storageClass: 'NEARLINE'}), + headers: {'Content-Type': 'application/json'}, + queryParameters: {}, + }; + if (options.preconditionRequired) { - await options.file!.setStorageClass('nearline', { - preconditionOpts: { - ifGenerationMatch: options.file!.metadata.generation, - }, - }); - } else { - await options.file!.setStorageClass('nearline'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const instanceOpts = (options.file as any)?.instancePreconditionOpts; + const generation = + instanceOpts?.ifGenerationMatch || options.file?.metadata?.generation; + requestOptions.queryParameters!.ifSourceGenerationMatch = generation; } + + return await options.storageTransport!.makeRequest(requestOptions); } // ///////////////////////////////////////////////// @@ -723,26 +1128,49 @@ export async function setStorageClass(options: ConformanceTestOptions) { // ///////////////////////////////////////////////// export async function deleteHMAC(options: ConformanceTestOptions) { - const metadata = { - state: 'INACTIVE', - }; - await options.hmacKey!.setMetadata(metadata); - await options.hmacKey!.delete(); + await options.storageTransport!.makeRequest({ + method: 'PUT', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + body: JSON.stringify({state: 'INACTIVE'}), + headers: {'x-retry-test-id': ''}, + }); + return await options.storageTransport!.makeRequest({ + method: 'DELETE', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + }); } export async function getHMAC(options: ConformanceTestOptions) { - await options.hmacKey!.get(); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + }); } export async function getMetadataHMAC(options: ConformanceTestOptions) { - await options.hmacKey!.getMetadata(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function setMetadataHMAC(options: ConformanceTestOptions) { - const metadata = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const body: any = { state: 'INACTIVE', }; - await options.hmacKey!.setMetadata(metadata); + + if (options.preconditionRequired && options.hmacKey?.metadata?.etag) { + body.etag = options.hmacKey.metadata.etag; + } + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: `storage/v1/projects/${options.projectId}/hmacKeys/${options.hmacKey!.metadata.accessId}`, + body: JSON.stringify(body), + }); } ///////////////////////////////////////////////// @@ -750,11 +1178,17 @@ export async function setMetadataHMAC(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function iamGetPolicy(options: ConformanceTestOptions) { - await options.bucket!.iam.getPolicy({requestedPolicyVersion: 1}); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam`, + queryParameters: {optionsRequestedPolicyVersion: 1}, + }); } export async function iamSetPolicy(options: ConformanceTestOptions) { - const testPolicy: Policy = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + const body: Policy = { bindings: [ { role: 'roles/storage.admin', @@ -762,16 +1196,40 @@ export async function iamSetPolicy(options: ConformanceTestOptions) { }, ], }; + if (options.preconditionRequired) { - const currentPolicy = await options.bucket!.iam.getPolicy(); - testPolicy.etag = currentPolicy[0].etag; + const getResponse = await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam`, + queryParameters: {optionsRequestedPolicyVersion: 1}, + headers: retryId ? {'x-retry-test-id': retryId} : {}, + }); + + const currentPolicy = getResponse as Policy; + const fetchedEtag = currentPolicy.etag; + + if (fetchedEtag) { + body.etag = fetchedEtag; + } } - await options.bucket!.iam.setPolicy(testPolicy); + + return await options.storageTransport!.makeRequest({ + method: 'PUT', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam`, + body: JSON.stringify(body), + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }); } export async function iamTestPermissions(options: ConformanceTestOptions) { - const permissionToTest = 'storage.buckets.delete'; - await options.bucket!.iam.testPermissions(permissionToTest); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/iam/testPermissions`, + queryParameters: {permissions: 'storage.buckets.delete'}, + }); } ///////////////////////////////////////////////// @@ -779,23 +1237,55 @@ export async function iamTestPermissions(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function notificationDelete(options: ConformanceTestOptions) { - await options.notification!.delete(); + const requestOptions: StorageRequestOptions = { + method: 'DELETE', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs/${options.notification!.id}`, + queryParameters: {}, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function notificationCreate(options: ConformanceTestOptions) { - await options.notification!.create(); + const requestOptions: StorageRequestOptions = { + method: 'POST', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs`, + body: JSON.stringify({ + topic: 'my-topic', + }), + headers: {'Content-Type': 'application/json'}, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function notificationExists(options: ConformanceTestOptions) { - await options.notification!.exists(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs/${options.notification!.id}`, + }; + + try { + await options.storageTransport!.makeRequest(requestOptions); + return true; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if (err.code === 404) return false; + throw err; + } } export async function notificationGet(options: ConformanceTestOptions) { - await options.notification!.get(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(options.bucket!.name)}/notificationConfigs/${options.notification!.id}`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } export async function notificationGetMetadata(options: ConformanceTestOptions) { - await options.notification!.getMetadata(); + return await notificationGet(options); } ///////////////////////////////////////////////// @@ -803,43 +1293,74 @@ export async function notificationGetMetadata(options: ConformanceTestOptions) { ///////////////////////////////////////////////// export async function createBucket(options: ConformanceTestOptions) { - const bucket = options.storage!.bucket('test-creating-bucket'); - const [exists] = await bucket.exists(); - if (exists) { - await bucket.delete(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const retryId = (options as any).headers?.['x-retry-test-id']; + const bucketName = options.bucket!.name; + + try { + return await options.storageTransport!.makeRequest({ + method: 'POST', + url: 'storage/v1/b', + queryParameters: {project: options.projectId}, + body: JSON.stringify({name: bucketName}), + headers: { + 'Content-Type': 'application/json', + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } catch (err: any) { + if ( + err.response?.status === 409 || + err.message?.includes('already exists') + ) { + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/b/${encodeURIComponent(bucketName)}`, + headers: { + ...(retryId ? {'x-retry-test-id': retryId} : {}), + }, + }); + } + throw err; } - await options.storage!.createBucket('test-creating-bucket'); } export async function createHMACKey(options: ConformanceTestOptions) { const serviceAccountEmail = 'my-service-account@appspot.gserviceaccount.com'; - await options.storage!.createHmacKey(serviceAccountEmail); + return await options.storageTransport!.makeRequest({ + method: 'POST', + url: `storage/v1/projects/${options.projectId}/hmacKeys`, + queryParameters: {serviceAccountEmail}, + }); } export async function getBuckets(options: ConformanceTestOptions) { - await options.storage!.getBuckets(); + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: 'storage/v1/b', + queryParameters: {project: options.projectId}, + }); } export async function getBucketsStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .storage!.getBucketsStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', err => reject(err)); - }); + return await getBuckets(options); } -export function getHMACKeyStream(options: ConformanceTestOptions) { - return new Promise((resolve, reject) => { - options - .storage!.getHmacKeysStream() - .on('data', () => {}) - .on('end', () => resolve(undefined)) - .on('error', err => reject(err)); +export async function getHMACKeyStream(options: ConformanceTestOptions) { + const serviceAccountEmail = 'my-service-account@appspot.gserviceaccount.com'; + return await options.storageTransport!.makeRequest({ + method: 'GET', + url: `storage/v1/projects/${options.projectId}/hmacKeys`, + queryParameters: {serviceAccountEmail}, }); } export async function getServiceAccount(options: ConformanceTestOptions) { - await options.storage!.getServiceAccount(); + const requestOptions: StorageRequestOptions = { + method: 'GET', + url: `storage/v1/projects/${options.projectId}/serviceAccount`, + }; + + return await options.storageTransport!.makeRequest(requestOptions); } diff --git a/handwritten/storage/conformance-test/test-data/retryInvocationMap.json b/handwritten/storage/conformance-test/test-data/retryInvocationMap.json index 8dea345f12c3..c9a52c43649a 100644 --- a/handwritten/storage/conformance-test/test-data/retryInvocationMap.json +++ b/handwritten/storage/conformance-test/test-data/retryInvocationMap.json @@ -44,7 +44,7 @@ "storage.notifications.list": [ "getNotifications" ], - "storage.buckets.lockRententionPolicy": [ + "storage.buckets.lockRetentionPolicy": [ "lock" ], "storage.objects.patch": [ @@ -134,6 +134,7 @@ "getMetadataHMAC" ], "storage.hmacKey.update": [ + "setMetadataHMAC" ], "storage.hmacKey.create": [ "createHMACKey" diff --git a/handwritten/storage/conformance-test/testBenchUtil.ts b/handwritten/storage/conformance-test/testBenchUtil.ts index b66f83094588..c231c86cf0b2 100644 --- a/handwritten/storage/conformance-test/testBenchUtil.ts +++ b/handwritten/storage/conformance-test/testBenchUtil.ts @@ -22,7 +22,7 @@ const PORT = new URL(HOST).port; const CONTAINER_NAME = 'storage-testbench'; const DEFAULT_IMAGE_NAME = 'gcr.io/cloud-devrel-public-resources/storage-testbench'; -const DEFAULT_IMAGE_TAG = 'v0.35.0'; +const DEFAULT_IMAGE_TAG = 'v0.60.0'; const DOCKER_IMAGE = `${DEFAULT_IMAGE_NAME}:${DEFAULT_IMAGE_TAG}`; const PULL_CMD = `docker pull ${DOCKER_IMAGE}`; const RUN_CMD = `docker run --rm -d -p ${PORT}:${PORT} --name ${CONTAINER_NAME} ${DOCKER_IMAGE} && sleep 1`;