From 5fc2a41f4ae1759f72c2bfcb95f63e85feb3aa88 Mon Sep 17 00:00:00 2001 From: yoshi-code-bot <70984784+yoshi-code-bot@users.noreply.github.com> Date: Wed, 14 May 2025 08:25:57 -0700 Subject: [PATCH 1/5] chore: update types from Discovery (#1476) Co-authored-by: Owl Bot --- src/types.d.ts | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/types.d.ts b/src/types.d.ts index 5d3a6b62..84cb4bd6 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -13,7 +13,7 @@ // limitations under the License. /** - * Discovery Revision: 20250313 + * Discovery Revision: 20250511 */ /** @@ -417,7 +417,7 @@ declare namespace bigquery { }; /** - * Configuration for BigLake managed tables. + * Configuration for BigQuery tables for Apache Iceberg (formerly BigLake managed tables.) */ type IBigLakeConfiguration = { /** @@ -1093,6 +1093,10 @@ declare namespace bigquery { * The dataset reference. Use this property to access specific parts of the dataset's ID, such as project ID or dataset ID. */ datasetReference?: IDatasetReference; + /** + * Output only. Reference to a read-only external dataset defined in data catalogs outside of BigQuery. Filled out when the dataset type is EXTERNAL. + */ + externalDatasetReference?: IExternalDatasetReference; /** * An alternate name for the dataset. The friendly name is purely decorative in nature. */ @@ -2549,7 +2553,7 @@ declare namespace bigquery { */ timePartitioning?: ITimePartitioning; /** - * Optional. [Experimental] Default time zone that will apply when parsing timestamp values that have no specific time zone. + * Optional. Default time zone that will apply when parsing timestamp values that have no specific time zone. */ timeZone?: string; /** @@ -2561,7 +2565,7 @@ declare namespace bigquery { */ useAvroLogicalTypes?: boolean; /** - * Optional. Specifies the action that occurs if the destination table already exists. The following values are supported: * WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the data, removes the constraints and uses the schema from the load job. * WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. * WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_APPEND. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion. + * Optional. Specifies the action that occurs if the destination table already exists. The following values are supported: * WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the data, removes the constraints and uses the schema from the load job. * WRITE_TRUNCATE_DATA: If the table already exists, BigQuery overwrites the data, but keeps the constraints and schema of the existing table. * WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. * WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_APPEND. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion. */ writeDisposition?: string; }; @@ -2675,7 +2679,7 @@ declare namespace bigquery { */ userDefinedFunctionResources?: Array; /** - * Optional. Specifies the action that occurs if the destination table already exists. The following values are supported: * WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the data, removes the constraints, and uses the schema from the query result. * WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. * WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_EMPTY. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion. + * Optional. Specifies the action that occurs if the destination table already exists. The following values are supported: * WRITE_TRUNCATE: If the table already exists, BigQuery overwrites the data, removes the constraints, and uses the schema from the query result. * WRITE_TRUNCATE_DATA: If the table already exists, BigQuery overwrites the data, but keeps the constraints and schema of the existing table. * WRITE_APPEND: If the table already exists, BigQuery appends the data to the table. * WRITE_EMPTY: If the table already exists and contains data, a 'duplicate' error is returned in the job result. The default value is WRITE_EMPTY. Each action is atomic and only occurs if BigQuery is able to complete the job successfully. Creation, truncation and append actions occur as one atomic update upon job completion. */ writeDisposition?: string; /** @@ -3055,7 +3059,7 @@ declare namespace bigquery { */ referencedRoutines?: Array; /** - * Output only. Referenced tables for the job. Queries that reference more than 50 tables will not have a complete list. + * Output only. Referenced tables for the job. */ referencedTables?: Array; /** @@ -4149,6 +4153,10 @@ declare namespace bigquery { * Total units of work remaining for the query. This number can be revised (increased or decreased) while the query is running. */ pendingUnits?: string; + /** + * Total shuffle usage ratio in shuffle RAM per reservation of this query. This will be provided for reservation customers only. + */ + shuffleRamUsageRatio?: number; /** * Cumulative slot-ms consumed by the query. */ @@ -4954,7 +4962,7 @@ declare namespace bigquery { type ITable = { /** - * Optional. Specifies the configuration of a BigLake managed table. + * Optional. Specifies the configuration of a BigQuery table for Apache Iceberg. */ biglakeConfiguration?: IBigLakeConfiguration; /** @@ -6390,6 +6398,14 @@ declare namespace bigquery { * Optional. The version of the provided access policy schema. Valid values are 0, 1, and 3. Requests specifying an invalid value will be rejected. This version refers to the schema version of the access policy and not the version of access policy. This field's value can be equal or more than the access policy schema provided in the request. For example, * Operations updating conditional access policy binding in datasets must specify version 3. Some of the operations are : - Adding a new access policy entry with condition. - Removing an access policy entry with condition. - Updating an access policy entry with condition. * But dataset with no conditional role bindings in access policy may specify any valid value or leave the field unset. If unset or if 0 or 1 value is used for dataset with conditional bindings, request will be rejected. This field will be mapped to IAM Policy version (https://cloud.google.com/iam/docs/policies#versions) and will be used to set policy in IAM. */ accessPolicyVersion?: number; + /** + * Optional. Specifies the fields of dataset that update/patch operation is targeting By default, both metadata and ACL fields are updated. + */ + updateMode?: + | 'UPDATE_MODE_UNSPECIFIED' + | 'UPDATE_METADATA' + | 'UPDATE_ACL' + | 'UPDATE_FULL'; }; /** @@ -6400,6 +6416,14 @@ declare namespace bigquery { * Optional. The version of the provided access policy schema. Valid values are 0, 1, and 3. Requests specifying an invalid value will be rejected. This version refers to the schema version of the access policy and not the version of access policy. This field's value can be equal or more than the access policy schema provided in the request. For example, * Operations updating conditional access policy binding in datasets must specify version 3. Some of the operations are : - Adding a new access policy entry with condition. - Removing an access policy entry with condition. - Updating an access policy entry with condition. * But dataset with no conditional role bindings in access policy may specify any valid value or leave the field unset. If unset or if 0 or 1 value is used for dataset with conditional bindings, request will be rejected. This field will be mapped to IAM Policy version (https://cloud.google.com/iam/docs/policies#versions) and will be used to set policy in IAM. */ accessPolicyVersion?: number; + /** + * Optional. Specifies the fields of dataset that update/patch operation is targeting By default, both metadata and ACL fields are updated. + */ + updateMode?: + | 'UPDATE_MODE_UNSPECIFIED' + | 'UPDATE_METADATA' + | 'UPDATE_ACL' + | 'UPDATE_FULL'; }; } From 8151e72bb1e149f6f36f7acdba25629d208b1074 Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Tue, 27 May 2025 16:59:19 -0400 Subject: [PATCH 2/5] feat: support per-job reservation assignment (#1477) --- src/bigquery.ts | 23 +++++++++++++++++++-- src/model.ts | 10 +++++++--- src/table.ts | 43 +++++++++++++++++++++++++++++---------- test/bigquery.ts | 37 ++++++++++++++++++++++++++++++++++ test/model.ts | 19 ++++++++++++++++++ test/table.ts | 52 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 168 insertions(+), 16 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index fb85ac09..cac13716 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -76,6 +76,7 @@ export type JobRequest = J & { jobPrefix?: string; location?: string; projectId?: string; + reservation?: string; }; export type PagedRequest

= P & { @@ -114,6 +115,7 @@ export type Query = JobRequest & { job?: Job; maxResults?: number; jobTimeoutMs?: number; + reservation?: string; pageToken?: string; wrapIntegers?: boolean | IntegerTypeCastOptions; parseJSON?: boolean; @@ -1562,6 +1564,11 @@ export class BigQuery extends Service { delete query.jobId; } + if (query.reservation) { + reqOpts.configuration.reservation = query.reservation; + delete query.reservation; + } + this.createJob(reqOpts, callback!); } @@ -1731,11 +1738,16 @@ export class BigQuery extends Service { location: this.location, }; - if (options.location) { - reqOpts.jobReference.location = options.location; + if (reqOpts.location) { + reqOpts.jobReference.location = reqOpts.location; delete reqOpts.location; } + if (reqOpts.configuration && reqOpts.reservation) { + reqOpts.configuration.reservation = reqOpts.reservation; + delete reqOpts.reservation; + } + const job = this.job(jobId!, { location: reqOpts.jobReference.location, }); @@ -2327,6 +2339,13 @@ export class BigQuery extends Service { useLegacySql: false, requestId: randomUUID(), jobCreationMode: 'JOB_CREATION_OPTIONAL', + reservation: queryObj.reservation, + continuous: queryObj.continuous, + destinationEncryptionConfiguration: + queryObj.destinationEncryptionConfiguration, + writeIncrementalResults: queryObj.writeIncrementalResults, + connectionProperties: queryObj.connectionProperties, + preserveNulls: queryObj.preserveNulls, }; if (!this._enableQueryPreview) { delete req.jobCreationMode; diff --git a/src/model.ts b/src/model.ts index 3c460e43..a0ac6ce3 100644 --- a/src/model.ts +++ b/src/model.ts @@ -26,7 +26,7 @@ import { RequestCallback, JobRequest, } from '.'; -import {JobMetadata} from './job'; +import {JobMetadata, JobOptions} from './job'; import bigquery from './types'; // This is supposed to be a @google-cloud/storage `File` type. The storage npm @@ -424,8 +424,7 @@ class Model extends ServiceObject { } } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { extract: extend(true, options, { sourceModel: { @@ -447,6 +446,11 @@ class Model extends ServiceObject { delete options.jobId; } + if (body.configuration && options.reservation) { + body.configuration.reservation = options.reservation; + delete options.reservation; + } + this.bigQuery.createJob(body, callback!); } diff --git a/src/table.ts b/src/table.ts index c3154b1e..ce436545 100644 --- a/src/table.ts +++ b/src/table.ts @@ -52,7 +52,7 @@ import { } from '.'; import {GoogleErrorBody} from '@google-cloud/common/build/src/util'; import {Duplex, Writable} from 'stream'; -import {JobMetadata} from './job'; +import {JobMetadata, JobOptions} from './job'; import bigquery from './types'; import {IntegerTypeCastOptions} from './bigquery'; import {RowQueue} from './rowQueue'; @@ -923,8 +923,7 @@ class Table extends ServiceObject { const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { copy: extend(true, metadata, { destinationTable: { @@ -955,6 +954,11 @@ class Table extends ServiceObject { delete metadata.jobId; } + if (body.configuration && metadata.reservation) { + body.configuration.reservation = metadata.reservation; + delete metadata.reservation; + } + this.bigQuery.createJob(body, callback!); } @@ -1045,8 +1049,7 @@ class Table extends ServiceObject { const callback = typeof metadataOrCallback === 'function' ? metadataOrCallback : cb; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { copy: extend(true, metadata, { destinationTable: { @@ -1080,6 +1083,11 @@ class Table extends ServiceObject { delete metadata.jobId; } + if (body.configuration && metadata.reservation) { + body.configuration.reservation = metadata.reservation; + delete metadata.reservation; + } + this.bigQuery.createJob(body, callback!); } @@ -1218,8 +1226,7 @@ class Table extends ServiceObject { delete options.gzip; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { extract: extend(true, options, { sourceTable: { @@ -1245,6 +1252,11 @@ class Table extends ServiceObject { delete options.jobId; } + if (body.configuration && options.reservation) { + body.configuration.reservation = options.reservation; + delete options.reservation; + } + this.bigQuery.createJob(body, callback!); } @@ -1399,8 +1411,7 @@ class Table extends ServiceObject { return [jobResponse, jobResponse.metadata]; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const body: any = { + const body: JobOptions = { configuration: { load: { destinationTable: { @@ -1427,7 +1438,12 @@ class Table extends ServiceObject { delete metadata.jobId; } - extend(true, body.configuration.load, metadata, { + if (body.configuration && metadata.reservation) { + body.configuration.reservation = metadata.reservation; + delete metadata.reservation; + } + + extend(true, body.configuration?.load, metadata, { sourceUris: toArray(source).map(src => { if (!util.isCustomType(src, 'storage/file')) { throw new Error('Source must be a File object.'); @@ -1437,7 +1453,12 @@ class Table extends ServiceObject { // the file's extension. If no match, don't set, and default upstream // to CSV. const format = FORMATS[path.extname(src.name).substr(1).toLowerCase()]; - if (!metadata.sourceFormat && format) { + if ( + !metadata.sourceFormat && + format && + body.configuration && + body.configuration.load + ) { body.configuration.load.sourceFormat = format; } return 'gs://' + src.bucket.name + '/' + src.name; diff --git a/test/bigquery.ts b/test/bigquery.ts index d63cdbb4..ab3e65ff 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -2644,6 +2644,20 @@ describe('BigQuery', () => { bq.createQueryJob(options, assert.ifError); }); + it('should accept a reservation id', done => { + const options = { + query: QUERY_STRING, + reservation: 'reservation/1', + }; + + bq.createJob = (reqOpts: JobOptions) => { + assert.strictEqual(reqOpts.configuration?.reservation, 'reservation/1'); + done(); + }; + + bq.createQueryJob(options, assert.ifError); + }); + it('should accept a location', done => { const options = { query: QUERY_STRING, @@ -3300,6 +3314,29 @@ describe('BigQuery', () => { bq.query(QUERY_STRING, fakeOptions, assert.ifError); }); + + it('should accept a reservation id', done => { + const query: Query = { + query: QUERY_STRING, + reservation: 'reservation/1', + }; + const fakeJob = { + getQueryResults: (options: {}) => { + done(); + }, + }; + + bq.createJob = (reqOpts: JobOptions, callback: Function) => { + assert(reqOpts.configuration?.reservation, 'reservation/1'); + callback(null, fakeJob, FAKE_RESPONSE); + }; + + bq.buildQueryRequest_ = (query: {}, opts: {}) => { + return undefined; + }; + + bq.query(query, assert.ifError); + }); }); describe('buildQueryRequest_', () => { diff --git a/test/model.ts b/test/model.ts index 5408e11d..49c59a5c 100644 --- a/test/model.ts +++ b/test/model.ts @@ -257,6 +257,25 @@ describe('BigQuery/Model', () => { model.createExtractJob(URI, options, done); }); + it('should accept a reservation id', done => { + const options = { + reservation: 'reservation/1', + }; + + model.bigQuery.createJob = ( + reqOpts: JobOptions, + callback: Function, + ) => { + assert.strictEqual( + reqOpts.configuration?.reservation, + 'reservation/1', + ); + callback(); // the done fn + }; + + model.createExtractJob(URI, options, done); + }); + it('should accept a job id', done => { const jobId = 'job-id'; const options = {jobId}; diff --git a/test/table.ts b/test/table.ts index 710d4cf1..e69b064b 100644 --- a/test/table.ts +++ b/test/table.ts @@ -767,6 +767,19 @@ describe('BigQuery/Table', () => { table.createCopyJob(DEST_TABLE, options, done); }); + it('should accept a reservation id', done => { + const options = { + reservation: 'reservation/1', + }; + + table.bigQuery.createJob = (reqOpts: JobOptions, callback: Function) => { + assert.strictEqual(reqOpts.configuration?.reservation, 'reservation/1'); + callback(); // the done fn + }; + + table.createCopyJob(DEST_TABLE, options, done); + }); + it('should use the default location', done => { table.bigQuery.createJob = (reqOpts: JobOptions, callback: Function) => { assert.strictEqual(reqOpts.location, LOCATION); @@ -911,6 +924,19 @@ describe('BigQuery/Table', () => { table.createCopyFromJob(SOURCE_TABLE, options, done); }); + it('should accept a reservation id', done => { + const options = { + reservation: 'reservation/1', + }; + + table.bigQuery.createJob = (reqOpts: JobOptions, callback: Function) => { + assert.strictEqual(reqOpts.configuration?.reservation, 'reservation/1'); + callback(); // the done fn + }; + + table.createCopyFromJob(SOURCE_TABLE, options, done); + }); + it('should use the default location', done => { table.bigQuery.createJob = (reqOpts: JobOptions, callback: Function) => { assert.strictEqual(reqOpts.location, LOCATION); @@ -1195,6 +1221,19 @@ describe('BigQuery/Table', () => { table.createExtractJob(FILE, options, done); }); + it('should accept a reservation id', done => { + const options = { + reservation: 'reservation/1', + }; + + table.bigQuery.createJob = (reqOpts: JobOptions, callback: Function) => { + assert.strictEqual(reqOpts.configuration?.reservation, 'reservation/1'); + callback(); // the done fn + }; + + table.createExtractJob(FILE, options, done); + }); + it('should use the default location', done => { const table = new Table(DATASET, TABLE_ID, {location: LOCATION}); @@ -1425,6 +1464,19 @@ describe('BigQuery/Table', () => { ); }); + it('should set the job reservation', async () => { + const reservation = 'reservation/1'; + await table.createLoadJob(FILE, {reservation}); + assert(bqCreateJobStub.calledOnce); + assert( + bqCreateJobStub.calledWithMatch({ + configuration: { + reservation, + }, + }), + ); + }); + it('should use the default location', async () => { const table = new Table(DATASET, TABLE_ID, {location: LOCATION}); await table.createLoadJob(FILE); From b51359a61d93a5d9cff729221f457a50a5c7a52f Mon Sep 17 00:00:00 2001 From: Alvaro Viebrantz Date: Wed, 28 May 2025 12:06:15 -0400 Subject: [PATCH 3/5] feat: job creation mode GA (#1480) --- README.md | 2 +- samples/README.md | 36 +++++++++---------- ...{queryShortMode.js => queryJobOptional.js} | 21 ++++++----- samples/test/queries.test.js | 4 +-- src/bigquery.ts | 34 +++++++++--------- test/bigquery.ts | 9 +++-- 6 files changed, 58 insertions(+), 48 deletions(-) rename samples/{queryShortMode.js => queryJobOptional.js} (76%) diff --git a/README.md b/README.md index 01effc94..7dc948f2 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-bigquery/tr | Query Dry Run | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryDryRun.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryDryRun.js,samples/README.md) | | Query External GCS Perm | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryExternalGCSPerm.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryExternalGCSPerm.js,samples/README.md) | | Query External GCS Temp | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryExternalGCSTemp.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryExternalGCSTemp.js,samples/README.md) | +| Query Job Optional | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryJobOptional.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryJobOptional.js,samples/README.md) | | Query Legacy | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryLegacy.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryLegacy.js,samples/README.md) | | Query Legacy Large Results | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryLegacyLargeResults.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryLegacyLargeResults.js,samples/README.md) | | Query Pagination | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryPagination.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryPagination.js,samples/README.md) | @@ -163,7 +164,6 @@ Samples are in the [`samples/`](https://github.com/googleapis/nodejs-bigquery/tr | Query Params Positional Types | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryParamsPositionalTypes.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryParamsPositionalTypes.js,samples/README.md) | | Query Params Structs | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryParamsStructs.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryParamsStructs.js,samples/README.md) | | Query Params Timestamps | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryParamsTimestamps.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryParamsTimestamps.js,samples/README.md) | -| Query Short Mode | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryShortMode.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryShortMode.js,samples/README.md) | | Query Stack Overflow | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryStackOverflow.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryStackOverflow.js,samples/README.md) | | Quickstart | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/quickstart.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/quickstart.js,samples/README.md) | | Relax Column | [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/relaxColumn.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/relaxColumn.js,samples/README.md) | diff --git a/samples/README.md b/samples/README.md index 57e38fa7..b94f02ac 100644 --- a/samples/README.md +++ b/samples/README.md @@ -86,6 +86,7 @@ * [Query Dry Run](#query-dry-run) * [Query External GCS Perm](#query-external-gcs-perm) * [Query External GCS Temp](#query-external-gcs-temp) + * [Query Job Optional](#query-job-optional) * [Query Legacy](#query-legacy) * [Query Legacy Large Results](#query-legacy-large-results) * [Query Pagination](#query-pagination) @@ -96,7 +97,6 @@ * [Query Params Positional Types](#query-params-positional-types) * [Query Params Structs](#query-params-structs) * [Query Params Timestamps](#query-params-timestamps) - * [Query Short Mode](#query-short-mode) * [Query Stack Overflow](#query-stack-overflow) * [Quickstart](#quickstart) * [Relax Column](#relax-column) @@ -1408,6 +1408,23 @@ __Usage:__ +### Query Job Optional + +View the [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryJobOptional.js). + +[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryJobOptional.js,samples/README.md) + +__Usage:__ + + +`node samples/queryJobOptional.js` + + +----- + + + + ### Query Legacy View the [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryLegacy.js). @@ -1578,23 +1595,6 @@ __Usage:__ -### Query Short Mode - -View the [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryShortMode.js). - -[![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/nodejs-bigquery&page=editor&open_in_editor=samples/queryShortMode.js,samples/README.md) - -__Usage:__ - - -`node samples/queryShortMode.js` - - ------ - - - - ### Query Stack Overflow View the [source code](https://github.com/googleapis/nodejs-bigquery/blob/main/samples/queryStackOverflow.js). diff --git a/samples/queryShortMode.js b/samples/queryJobOptional.js similarity index 76% rename from samples/queryShortMode.js rename to samples/queryJobOptional.js index 2ca0e309..33cf5b4e 100644 --- a/samples/queryShortMode.js +++ b/samples/queryJobOptional.js @@ -15,16 +15,17 @@ 'use strict'; function main() { - // [START bigquery_query_shortmode] + // [START bigquery_query_job_optional] // Demonstrates issuing a query that may be run in short query mode. - // To enable the short query mode preview feature, the QUERY_PREVIEW_ENABLED - // environmental variable should be set to `TRUE`. // Import the Google Cloud client library const {BigQuery} = require('@google-cloud/bigquery'); - const bigquery = new BigQuery(); + const bigquery = new BigQuery({ + // default behavior is to create jobs when using the jobs.query API + defaultJobCreationMode: 'JOB_CREATION_REQUIRED', + }); - async function queryShortMode() { + async function queryJobOptional() { // SQL query to run. const sqlQuery = ` @@ -35,7 +36,11 @@ function main() { LIMIT 10`; // Run the query - const [rows, , res] = await bigquery.query(sqlQuery); + const [rows, , res] = await bigquery.query({ + query: sqlQuery, + // Skip job creation to enable short mode. + jobCreationMode: 'JOB_CREATION_OPTIONAL', + }); if (!res.jobReference) { console.log(`Query was run in short mode. Query ID: ${res.queryId}`); @@ -50,7 +55,7 @@ function main() { console.log('Rows:'); rows.forEach(row => console.log(row)); } - // [END bigquery_query_shortmode] - queryShortMode(); + // [END bigquery_query_job_optional] + queryJobOptional(); } main(...process.argv.slice(2)); diff --git a/samples/test/queries.test.js b/samples/test/queries.test.js index 472dd88f..3ef02a47 100644 --- a/samples/test/queries.test.js +++ b/samples/test/queries.test.js @@ -70,8 +70,8 @@ describe('Queries', () => { assert.match(output, /name/); }); - it('should run a query in short mode', async () => { - const output = execSync('node queryShortMode.js'); + it('should run a query stateless mode', async () => { + const output = execSync('node queryJobOptional.js'); assert.match(output, /Rows:/); assert.match(output, /name/); }); diff --git a/src/bigquery.ts b/src/bigquery.ts index cac13716..09b1f518 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -105,6 +105,8 @@ export type SimpleQueryRowsCallback = ResourceCallback< bigquery.IJob >; +type JobCreationMode = bigquery.IQueryRequest['jobCreationMode']; + export type Query = JobRequest & { destination?: Table; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -119,6 +121,8 @@ export type Query = JobRequest & { pageToken?: string; wrapIntegers?: boolean | IntegerTypeCastOptions; parseJSON?: boolean; + // Overrides default job creation mode set on the client. + jobCreationMode?: JobCreationMode; }; export type QueryParamTypeStruct = { @@ -271,6 +275,13 @@ export interface BigQueryOptions extends GoogleAuthOptions { * Defaults to `googleapis.com`. */ universeDomain?: string; + + /** + * Controls the job creation mode used when executing queries that can be + * accelerated via the jobs.Query API. Users may experience performance + * improvements by leveraging the JOB_CREATION_OPTIONAL mode. + */ + defaultJobCreationMode?: JobCreationMode; } export interface IntegerTypeCastOptions { @@ -320,12 +331,6 @@ export const PROTOCOL_REGEX = /^(\w*):\/\//; * We will create a table with the correct schema, import the public CSV file * into that table, and query it for data. * - * This client supports enabling query-related preview features via environmental - * variables. By setting the environment variable QUERY_PREVIEW_ENABLED to the string - * "TRUE", the client will enable preview features, though behavior may still be - * controlled via the bigquery service as well. Currently, the feature(s) in scope - * include: stateless queries (query execution without corresponding job metadata). - * * @class * * See {@link https://cloud.google.com/bigquery/what-is-bigquery| What is BigQuery?} @@ -362,7 +367,7 @@ export const PROTOCOL_REGEX = /^(\w*):\/\//; export class BigQuery extends Service { location?: string; private _universeDomain: string; - private _enableQueryPreview: boolean; + private _defaultJobCreationMode: JobCreationMode; createQueryStream(options?: Query | string): ResourceStream { // placeholder body, overwritten in constructor @@ -421,12 +426,8 @@ export class BigQuery extends Service { super(config, options); - const QUERY_PREVIEW_ENABLED = process.env.QUERY_PREVIEW_ENABLED; - this._enableQueryPreview = false; - if (typeof QUERY_PREVIEW_ENABLED === 'string') { - if (QUERY_PREVIEW_ENABLED.toUpperCase() === 'TRUE') { - this._enableQueryPreview = true; - } + if (options.defaultJobCreationMode) { + this._defaultJobCreationMode = options.defaultJobCreationMode; } this._universeDomain = universeDomain; @@ -2338,7 +2339,7 @@ export class BigQuery extends Service { query: queryObj.query, useLegacySql: false, requestId: randomUUID(), - jobCreationMode: 'JOB_CREATION_OPTIONAL', + jobCreationMode: this._defaultJobCreationMode, reservation: queryObj.reservation, continuous: queryObj.continuous, destinationEncryptionConfiguration: @@ -2347,8 +2348,9 @@ export class BigQuery extends Service { connectionProperties: queryObj.connectionProperties, preserveNulls: queryObj.preserveNulls, }; - if (!this._enableQueryPreview) { - delete req.jobCreationMode; + if (queryObj.jobCreationMode) { + // override default job creation mode + req.jobCreationMode = queryObj.jobCreationMode; } const {parameterMode, params} = this.buildQueryParams_( queryObj.params, diff --git a/test/bigquery.ts b/test/bigquery.ts index ab3e65ff..b53a2b89 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -189,8 +189,10 @@ describe('BigQuery', () => { beforeEach(() => { Object.assign(fakeUtil, originalFakeUtil); BigQuery = Object.assign(BigQuery, BigQueryCached); - bq = new BigQuery({projectId: PROJECT_ID}); - bq._enableQueryPreview = true; + bq = new BigQuery({ + projectId: PROJECT_ID, + defaultJobCreationMode: 'JOB_CREATION_OPTIONAL', + }); }); after(() => { @@ -3360,6 +3362,7 @@ describe('BigQuery', () => { labels: { key: 'value', }, + jobCreationMode: 'JOB_CREATION_REQUIRED', }; const req = bq.buildQueryRequest_(q, {}); for (const key in req) { @@ -3392,7 +3395,7 @@ describe('BigQuery', () => { labels: { key: 'value', }, - jobCreationMode: 'JOB_CREATION_OPTIONAL', + jobCreationMode: 'JOB_CREATION_REQUIRED', formatOptions: { useInt64Timestamp: true, }, From 1bd620ad96064a7d55855e263eb5917e226f3044 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Wed, 28 May 2025 21:20:29 +0200 Subject: [PATCH 4/5] chore(deps): update dependency google-auth-library to v10.0.0-rc.2 (#1478) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 85e3e73b..f111cdb2 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ }, "overrides": { "@google-cloud/common": { - "google-auth-library": "10.0.0-rc.1" + "google-auth-library": "10.0.0-rc.2" } }, "devDependencies": { From 9729d1e4494fc266b6effbe204b6ff1996511abc Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 29 May 2025 14:08:59 -0400 Subject: [PATCH 5/5] chore(main): release 8.1.0 (#1481) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 ++++++++ package.json | 2 +- samples/package.json | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89b365ab..b4238611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://www.npmjs.com/package/@google-cloud/bigquery?activeTab=versions +## [8.1.0](https://github.com/googleapis/nodejs-bigquery/compare/v8.0.0...v8.1.0) (2025-05-29) + + +### Features + +* Job creation mode GA ([#1480](https://github.com/googleapis/nodejs-bigquery/issues/1480)) ([b51359a](https://github.com/googleapis/nodejs-bigquery/commit/b51359a61d93a5d9cff729221f457a50a5c7a52f)) +* Support per-job reservation assignment ([#1477](https://github.com/googleapis/nodejs-bigquery/issues/1477)) ([8151e72](https://github.com/googleapis/nodejs-bigquery/commit/8151e72bb1e149f6f36f7acdba25629d208b1074)) + ## [8.0.0](https://github.com/googleapis/nodejs-bigquery/compare/v7.9.4...v8.0.0) (2025-04-23) diff --git a/package.json b/package.json index f111cdb2..6f1cb428 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@google-cloud/bigquery", "description": "Google BigQuery Client Library for Node.js", - "version": "8.0.0", + "version": "8.1.0", "license": "Apache-2.0", "author": "Google LLC", "engines": { diff --git a/samples/package.json b/samples/package.json index 9534c1bd..804cb0d0 100644 --- a/samples/package.json +++ b/samples/package.json @@ -17,7 +17,7 @@ "fix": "gts fix" }, "dependencies": { - "@google-cloud/bigquery": "^8.0.0", + "@google-cloud/bigquery": "^8.1.0", "@google-cloud/storage": "^7.0.0", "google-auth-library": "^9.6.0", "readline-promise": "^1.0.4",