diff --git a/gapic-libraries-bom/pom.xml b/gapic-libraries-bom/pom.xml index 329f86d25e5a..fadd93e25cf8 100644 --- a/gapic-libraries-bom/pom.xml +++ b/gapic-libraries-bom/pom.xml @@ -301,13 +301,6 @@ pom import - - com.google.cloud - google-cloud-bigtable-deps-bom - 2.78.1-SNAPSHOT - pom - import - com.google.cloud google-cloud-billing-bom diff --git a/java-bigquery/google-cloud-bigquery-jdbc/pom.xml b/java-bigquery/google-cloud-bigquery-jdbc/pom.xml index f75bcfb165b5..78cddc110046 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/pom.xml +++ b/java-bigquery/google-cloud-bigquery-jdbc/pom.xml @@ -377,6 +377,12 @@ opentelemetry-sdk-testing test + + com.google.cloud + google-cloud-trace + 2.92.0 + test + diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java index 287fbc8a70a8..3a2f1a728544 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryConnection.java @@ -363,7 +363,7 @@ String getConnectionUrl() { return connectionUrl; } - String getConnectionId() { + public String getConnectionId() { return this.connectionId; } diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java index 70c23f0582b4..6fe251f0e8fa 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryJdbcOpenTelemetry.java @@ -65,6 +65,11 @@ public class BigQueryJdbcOpenTelemetry { private static final String OTLP_ENDPOINT_VALUE = "https://telemetry.googleapis.com:443"; private static final String EXPORTER_NONE = "none"; private static final String EXPORTER_OTLP = "otlp"; + private static final String OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = + "otel.span.attribute.value.length.limit"; + private static final String OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT = + "otel.attribute.value.length.limit"; + private static final String DEFAULT_ATTRIBUTE_LENGTH_LIMIT = "32768"; private static final BigQueryJdbcCustomLogger LOG = new BigQueryJdbcCustomLogger("BigQueryJdbcOpenTelemetry"); @@ -290,6 +295,17 @@ public static OpenTelemetry getOpenTelemetry( props.put(GOOGLE_CLOUD_PROJECT, gcpTelemetryProjectId); } + // Set safe, generous default limits on attribute value lengths (32KB) to protect + // customers from GCP Cloud Trace 64KB span ingestion failures when logging massive + // exception stack traces or database schema metadata. + // Respect any existing user configuration overrides. + if (!props.containsKey(OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT)) { + props.put(OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, DEFAULT_ATTRIBUTE_LENGTH_LIMIT); + } + if (!props.containsKey(OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT)) { + props.put(OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, DEFAULT_ATTRIBUTE_LENGTH_LIMIT); + } + AutoConfiguredOpenTelemetrySdk autoConfigured = AutoConfiguredOpenTelemetrySdk.builder().addPropertiesSupplier(() -> props).build(); diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java index c8192cf47e0c..f652d91b3f2e 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/OpenTelemetryJulHandler.java @@ -42,7 +42,9 @@ public class OpenTelemetryJulHandler extends Handler { private static final Pattern UNSAFE_LOG_CHARACTERS = Pattern.compile("[^a-zA-Z0-9./_-]"); - public OpenTelemetryJulHandler() {} + public OpenTelemetryJulHandler() { + setLevel(Level.ALL); + } @Override public void publish(LogRecord record) { diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java index dd6ceb0deceb..3939ae8c9536 100644 --- a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryConnectionTest.java @@ -26,17 +26,26 @@ import com.google.cloud.bigquery.exception.BigQueryJdbcException; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.BigQueryWriteClient; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.testing.junit5.OpenTelemetryExtension; +import io.opentelemetry.sdk.trace.data.SpanData; import java.io.IOException; import java.io.InputStream; import java.sql.SQLException; +import java.util.List; import java.util.Properties; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; public class BigQueryConnectionTest { + @RegisterExtension + static final OpenTelemetryExtension otelTesting = OpenTelemetryExtension.create(); + private static final String DEFAULT_VERSION = "0.0.0"; private static final String DEFAULT_JDBC_TOKEN_VALUE = "Google-BigQuery-JDBC-Driver"; private static final String BASE_URL = @@ -456,4 +465,25 @@ public void testIsReadOnlyTokenProvided(String readonlyProp, boolean expectedIsR assertEquals(expectedIsReadOnly, connection.isReadOnlyTokenUsed()); } } + + @Test + public void testConnect_withCustomOpenTelemetry_usesCustomInstance() throws Exception { + DataSource ds = DataSource.fromUrl(BASE_URL); + ds.setCustomOpenTelemetry(otelTesting.getOpenTelemetry()); + + try (BigQueryConnection connection = new BigQueryConnection(BASE_URL, ds)) { + assertNotNull(connection); + assertFalse(connection.isClosed()); + + Tracer tracer = connection.getTracer(); + assertNotNull(tracer); + + Span span = tracer.spanBuilder("custom-otel-span").startSpan(); + span.end(); + + List spans = otelTesting.getSpans(); + assertEquals(1, spans.size()); + assertEquals("custom-otel-span", spans.get(0).getName()); + } + } } diff --git a/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java new file mode 100644 index 000000000000..16dca4ca69ed --- /dev/null +++ b/java-bigquery/google-cloud-bigquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/it/ITOpenTelemetryTest.java @@ -0,0 +1,297 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.bigquery.jdbc.it; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import com.google.api.gax.paging.Page; +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.jdbc.BigQueryConnection; +import com.google.cloud.logging.LogEntry; +import com.google.cloud.logging.Logging; +import com.google.cloud.logging.LoggingOptions; +import com.google.cloud.trace.v1.TraceServiceClient; +import com.google.devtools.cloudtrace.v1.Trace; +import com.google.devtools.cloudtrace.v1.TraceSpan; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +public class ITOpenTelemetryTest { + + private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); + private static final String CONNECTION_URL = + String.format( + "jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectId=%s;OAuthType=3;Timeout=3600;", + PROJECT_ID); + + @Test + public void testExecute_withOpenTelemetryGcpExporter() throws Exception { + assumeTrue( + PROJECT_ID != null && !PROJECT_ID.trim().isEmpty(), + "Skipping OTel E2E tests because no default Project ID is configured."); + + // Step 1: Connect with GCP Exporters enabled + Properties props = new Properties(); + props.setProperty("enableGcpTraceExporter", "true"); + props.setProperty("enableGcpLogExporter", "true"); + props.setProperty("LogLevel", "3"); // Triggers FINE log generation + props.setProperty("gcpTelemetryProjectId", PROJECT_ID); + props.setProperty("EnableHighThroughputAPI", "0"); + props.setProperty("MaxResults", "50"); // Forces small page size (50) to trigger pagination + + String connectionUuid = null; + + try (Connection connection = DriverManager.getConnection(CONNECTION_URL, props); + Statement statement = connection.createStatement()) { + + // Retrieve the Connection UUID programmatically + BigQueryConnection bqConnection = connection.unwrap(BigQueryConnection.class); + connectionUuid = bqConnection.getConnectionId(); + assertNotNull(connectionUuid, "Connection UUID should be generated"); + + // Execute an in-memory array query (scans 0 bytes, extremely fast) and force pagination + String paginationQuery = "SELECT * FROM UNNEST(GENERATE_ARRAY(1, 1000)) AS id;"; + try (ResultSet paginatedRs = statement.executeQuery(paginationQuery)) { + int rowCount = 0; + while (paginatedRs.next() && rowCount < 1000) { + rowCount++; + } + } + } + + // Step 2: Retrieve logs from Cloud Logging and extract TraceId + String traceId = null; + String hexSpanId = null; + + try (Logging logging = + LoggingOptions.newBuilder().setProjectId(PROJECT_ID).build().getService()) { + String filter = + "logName:\"projects/" + + PROJECT_ID + + "/logs/com.google.cloud.bigquery\" AND labels.\"jdbc.connection_id\"=\"" + + connectionUuid + + "\""; + + List entries = fetchLogsWithRetry(logging, filter); + assertFalse(entries.isEmpty(), "Telemetry logs should be exported to GCP"); + + LogEntry sampleEntry = entries.get(0); + traceId = sampleEntry.getTrace(); + hexSpanId = sampleEntry.getSpanId(); + + assertNotNull(traceId, "Log entry must contain TraceId"); + assertNotNull(hexSpanId, "Log entry must contain SpanId"); + + // Verify Connection UUID label correlation on all entries + for (LogEntry entry : entries) { + assertEquals(connectionUuid, entry.getLabels().get("jdbc.connection_id")); + } + } + + // Step 3: Query Cloud Trace using TraceId and assert parent-child hierarchy + String hexTraceId = traceId; + if (traceId.contains("/traces/")) { + hexTraceId = traceId.substring(traceId.lastIndexOf("/traces/") + 8); + } + + try (TraceServiceClient traceClient = TraceServiceClient.create()) { + Trace trace = fetchTraceWithRetry(traceClient, PROJECT_ID, hexTraceId); + assertNotNull(trace, "Trace must be found in Cloud Trace API: " + hexTraceId); + + boolean foundParentExecuteQuery = false; + boolean foundChildSdkSpans = false; + boolean foundPaginationSpans = false; + long parentSpanId = 0; + + for (TraceSpan span : trace.getSpansList()) { + String spanName = span.getName(); + if (spanName.equals("BigQueryStatement.executeQuery")) { + foundParentExecuteQuery = true; + parentSpanId = span.getSpanId(); + } + } + + assertTrue( + foundParentExecuteQuery, + "Traces must contain JDBC parent span 'BigQueryStatement.executeQuery'"); + + // Verify that we captured child spans or linked pagination spans + for (TraceSpan span : trace.getSpansList()) { + if (span.getParentSpanId() == parentSpanId && parentSpanId != 0) { + foundChildSdkSpans = true; + } + if (span.getName().equals("BigQueryStatement.pagination")) { + foundPaginationSpans = true; + } + } + + assertTrue(foundPaginationSpans, "OTel pagination must generate pagination spans"); + assertTrue( + foundChildSdkSpans, + "OTel context must propagate parent to downstream pagination child spans"); + } + } + + @Test + public void testExecute_withErrorCorrelation() throws Exception { + assumeTrue( + PROJECT_ID != null && !PROJECT_ID.trim().isEmpty(), + "Skipping OTel E2E tests because no default Project ID is configured."); + + // Step 1: Connect with GCP Exporters enabled + Properties props = new Properties(); + props.setProperty("enableGcpTraceExporter", "true"); + props.setProperty("enableGcpLogExporter", "true"); + props.setProperty("LogLevel", "3"); // Triggers FINE log generation + props.setProperty("gcpTelemetryProjectId", PROJECT_ID); + + String connectionUuid = null; + + try (Connection connection = DriverManager.getConnection(CONNECTION_URL, props); + Statement statement = connection.createStatement()) { + + // Retrieve the Connection UUID programmatically + BigQueryConnection bqConnection = connection.unwrap(BigQueryConnection.class); + connectionUuid = bqConnection.getConnectionId(); + assertNotNull(connectionUuid, "Connection UUID should be generated"); + + // Execute a query designed to fail due to non-existent table + boolean caughtException = false; + try { + statement.executeQuery("SELECT * FROM invalid_dataset.invalid_table;"); + } catch (SQLException e) { + caughtException = true; + } + assertTrue(caughtException, "Expected SQLException to be thrown"); + } + + // Step 2: Retrieve logs from Cloud Logging and assert error logs + String traceId = null; + String hexSpanId = null; + + try (Logging logging = + LoggingOptions.newBuilder().setProjectId(PROJECT_ID).build().getService()) { + String filter = + "logName:\"projects/" + + PROJECT_ID + + "/logs/com.google.cloud.bigquery\" AND labels.\"jdbc.connection_id\"=\"" + + connectionUuid + + "\""; + + List entries = fetchLogsWithRetry(logging, filter); + assertFalse(entries.isEmpty(), "Telemetry logs should be exported to GCP"); + + LogEntry sampleEntry = entries.get(0); + traceId = sampleEntry.getTrace(); + hexSpanId = sampleEntry.getSpanId(); + + assertNotNull(traceId, "Log entry must contain TraceId"); + assertNotNull(hexSpanId, "Log entry must contain SpanId"); + } + + // Step 3: Query Cloud Trace using TraceId and assert span status is ERROR + String hexTraceId = traceId; + if (traceId.contains("/traces/")) { + hexTraceId = traceId.substring(traceId.lastIndexOf("/traces/") + 8); + } + + try (TraceServiceClient traceClient = TraceServiceClient.create()) { + Trace trace = fetchTraceWithRetry(traceClient, PROJECT_ID, hexTraceId); + assertNotNull(trace, "Trace must be found in Cloud Trace API: " + hexTraceId); + + boolean foundParentExecuteQuery = false; + + for (TraceSpan span : trace.getSpansList()) { + String spanName = span.getName(); + if (spanName.equals("BigQueryStatement.executeQuery")) { + foundParentExecuteQuery = true; + } + } + + assertTrue( + foundParentExecuteQuery, + "Traces must contain JDBC parent span 'BigQueryStatement.executeQuery'"); + } + } + + private T pollWithRetry(java.util.concurrent.Callable task) throws InterruptedException { + int attempts = 0; + int maxAttempts = 30; // 30 attempts * 500ms = 15 seconds max delay + long delayMs = 500; // 500ms linear polling + + while (attempts < maxAttempts) { + attempts++; + Thread.sleep(delayMs); + try { + T result = task.call(); + if (result != null) { + return result; + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Test execution interrupted", e); + } catch (Exception e) { + // Ignore exceptions during remote lookup and retry + } + } + return null; + } + + private List fetchLogsWithRetry(Logging logging, String filter) + throws InterruptedException { + List result = + pollWithRetry( + () -> { + Page entriesPage = + logging.listLogEntries( + Logging.EntryListOption.filter(filter), Logging.EntryListOption.pageSize(50)); + List entries = new ArrayList<>(); + entriesPage.iterateAll().forEach(entries::add); + return entries.isEmpty() ? null : entries; + }); + return result != null ? result : new ArrayList<>(); + } + + private Trace fetchTraceWithRetry( + TraceServiceClient traceClient, String projectId, String traceId) + throws InterruptedException { + return pollWithRetry( + () -> { + Trace trace = traceClient.getTrace(projectId, traceId); + if (trace == null) { + return null; + } + for (TraceSpan span : trace.getSpansList()) { + if (span.getName().equals("BigQueryStatement.executeQuery")) { + return trace; + } + } + return null; + }); + } +}