From 388ce3daaab1ada8de33488f9973220cb05d6100 Mon Sep 17 00:00:00 2001 From: AssemblyAI Date: Mon, 27 Apr 2026 12:26:33 -0400 Subject: [PATCH] Project import generated by Copybara. GitOrigin-RevId: a2eda254b7c2230131085f055d86405d59da40da --- package.json | 2 +- src/services/streaming/service.ts | 81 +++++++++++++++++++++++++------ src/types/openapi.generated.ts | 21 ++++++++ src/types/streaming/index.ts | 17 ++++++- src/utils/errors/streaming.ts | 12 +++++ tests/unit/streaming.test.ts | 66 +++++++++++++++++++++++++ tests/unit/transcript.test.ts | 77 +++++++++++++++++++++++++++++ 7 files changed, 259 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 1c129eb..833ae4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assemblyai", - "version": "4.30.0", + "version": "4.32.1", "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.", "engines": { "node": ">=18" diff --git a/src/services/streaming/service.ts b/src/services/streaming/service.ts index 5cefccf..1f0c6ef 100644 --- a/src/services/streaming/service.ts +++ b/src/services/streaming/service.ts @@ -16,6 +16,7 @@ import { LLMGatewayResponseEvent, StreamingUpdateConfiguration, StreamingForceEndpoint, + WarningEvent, } from "../.."; import { StreamingError, StreamingErrorMessages } from "../../utils/errors"; import { StreamingErrorTypeCodes } from "../../utils/errors/streaming"; @@ -86,19 +87,22 @@ export class StreamingTranscriber { ); } - if (this.params.minTurnSilence) { - searchParams.set( - "min_turn_silence", - this.params.minTurnSilence.toString(), - ); - } else if (this.params.minEndOfTurnSilenceWhenConfident) { - console.warn( - "[Deprecation Warning] `minEndOfTurnSilenceWhenConfident` is deprecated and will be removed in a future release. Please use `minTurnSilence` instead.", - ); - searchParams.set( - "min_end_of_turn_silence_when_confident", - this.params.minEndOfTurnSilenceWhenConfident.toString(), - ); + if (this.params.minEndOfTurnSilenceWhenConfident !== undefined) { + if (this.params.minTurnSilence !== undefined) { + console.warn( + "[Deprecation Warning] Both `minEndOfTurnSilenceWhenConfident` and `minTurnSilence` are set. Using `minTurnSilence`; `minEndOfTurnSilenceWhenConfident` is deprecated.", + ); + } else { + console.warn( + "[Deprecation Warning] `minEndOfTurnSilenceWhenConfident` is deprecated and will be removed in a future release. Please use `minTurnSilence` instead.", + ); + } + } + const effectiveMinTurnSilence = + this.params.minTurnSilence ?? + this.params.minEndOfTurnSilenceWhenConfident; + if (effectiveMinTurnSilence !== undefined) { + searchParams.set("min_turn_silence", effectiveMinTurnSilence.toString()); } if (this.params.maxTurnSilence) { @@ -176,6 +180,20 @@ export class StreamingTranscriber { searchParams.set("max_speakers", this.params.maxSpeakers.toString()); } + if (this.params.noiseSuppressionModel) { + searchParams.set( + "noise_suppression_model", + this.params.noiseSuppressionModel, + ); + } + + if (this.params.noiseSuppressionThreshold !== undefined) { + searchParams.set( + "noise_suppression_threshold", + this.params.noiseSuppressionThreshold.toString(), + ); + } + if (this.params.llmGateway !== undefined) { searchParams.set("llm_gateway", JSON.stringify(this.params.llmGateway)); } @@ -191,6 +209,7 @@ export class StreamingTranscriber { event: "llmGatewayResponse", listener: (event: LLMGatewayResponseEvent) => void, ): void; + on(event: "warning", listener: (event: WarningEvent) => void): void; on(event: "error", listener: (error: Error) => void): void; on(event: "close", listener: (code: number, reason: string) => void): void; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -243,7 +262,12 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c const message = JSON.parse(data.toString()) as StreamingEventMessage; if ("error" in message) { - this.listeners.error?.(new StreamingError(message.error)); + const err = new StreamingError(message.error); + if ("error_code" in message) { + (err as StreamingError & { code?: number }).code = + message.error_code; + } + this.listeners.error?.(err); return; } @@ -265,6 +289,14 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.listeners.llmGatewayResponse?.(message); break; } + case "Warning": { + const warning = message as WarningEvent; + console.warn( + `Streaming warning (code=${warning.warning_code}): ${warning.warning}`, + ); + this.listeners.warning?.(warning); + break; + } case "Termination": { this.sessionTerminatedResolve?.(); break; @@ -291,9 +323,28 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c * @param config - The configuration parameters to update */ updateConfiguration(config: Omit) { + const { + min_end_of_turn_silence_when_confident, + min_turn_silence, + ...rest + } = config; + if (min_end_of_turn_silence_when_confident !== undefined) { + if (min_turn_silence !== undefined) { + console.warn( + "[Deprecation Warning] Both `min_end_of_turn_silence_when_confident` and `min_turn_silence` are set. Using `min_turn_silence`; `min_end_of_turn_silence_when_confident` is deprecated.", + ); + } else { + console.warn( + "[Deprecation Warning] `min_end_of_turn_silence_when_confident` is deprecated and will be removed in a future release. Please use `min_turn_silence` instead.", + ); + } + } + const effective = + min_turn_silence ?? min_end_of_turn_silence_when_confident; const message: StreamingUpdateConfiguration = { type: "UpdateConfiguration", - ...config, + ...rest, + ...(effective !== undefined ? { min_turn_silence: effective } : {}), }; this.send(JSON.stringify(message)); } diff --git a/src/types/openapi.generated.ts b/src/types/openapi.generated.ts index 33dd10e..3e439fb 100644 --- a/src/types/openapi.generated.ts +++ b/src/types/openapi.generated.ts @@ -2825,6 +2825,10 @@ export type Transcript = { * See {@link https://www.assemblyai.com/docs/models/pii-redaction | PII redaction } for more information. */ redact_pii_policies?: PiiPolicy[] | null; + /** + * Whether the unredacted text, words, and utterances were also returned alongside the redacted fields. Only applies when `redact_pii` is enabled. + */ + redact_pii_return_unredacted?: boolean | null; /** * The replacement logic for detected PII, can be "entity_type" or "hash". See {@link https://www.assemblyai.com/docs/models/pii-redaction | PII redaction } for more details. */ @@ -2917,6 +2921,18 @@ export type Transcript = { * The list of custom topics provided if custom topics is enabled */ topics?: string[]; + /** + * The unredacted transcript text. Returned only when `redact_pii_return_unredacted` was set with `redact_pii`. + */ + unredacted_text?: string | null; + /** + * The unredacted list of utterances. Returned only when `redact_pii_return_unredacted` was set with `redact_pii` and channel/speaker modes are enabled. + */ + unredacted_utterances?: TranscriptUtterance[] | null; + /** + * The unredacted list of individual words. Returned only when `redact_pii_return_unredacted` was set with `redact_pii`. + */ + unredacted_words?: TranscriptWord[] | null; /** * When dual_channel or speaker_labels is enabled, a list of turn-by-turn utterance objects. * See {@link https://www.assemblyai.com/docs/models/speaker-diarization | Speaker diarization } for more information. @@ -3396,6 +3412,11 @@ export type TranscriptOptionalParams = { * The list of PII Redaction policies to enable. See {@link https://www.assemblyai.com/docs/models/pii-redaction | PII redaction } for more details. */ redact_pii_policies?: PiiPolicy[]; + /** + * If `redact_pii` is enabled, also return the unredacted text, words, and utterances alongside the redacted fields. + * @defaultValue false + */ + redact_pii_return_unredacted?: boolean; /** * The replacement logic for detected PII, can be "entity_type" or "hash". See {@link https://www.assemblyai.com/docs/models/pii-redaction | PII redaction } for more details. * @defaultValue "hash" diff --git a/src/types/streaming/index.ts b/src/types/streaming/index.ts index 2591256..eccb8dc 100644 --- a/src/types/streaming/index.ts +++ b/src/types/streaming/index.ts @@ -36,6 +36,8 @@ export type StreamingTranscriberParams = { inactivityTimeout?: number; speakerLabels?: boolean; maxSpeakers?: number; + noiseSuppressionModel?: NoiseSuppressionModel; + noiseSuppressionThreshold?: number; llmGateway?: LLMGatewayConfig; }; @@ -45,6 +47,7 @@ export type StreamingEvents = | "turn" | "speechStarted" | "llmGatewayResponse" + | "warning" | "error"; export type StreamingListeners = { @@ -53,6 +56,7 @@ export type StreamingListeners = { turn?: (event: TurnEvent) => void; speechStarted?: (event: SpeechStartedEvent) => void; llmGatewayResponse?: (event: LLMGatewayResponseEvent) => void; + warning?: (event: WarningEvent) => void; error?: (error: Error) => void; }; @@ -65,6 +69,8 @@ export type StreamingSpeechModel = export type StreamingDomain = "medical-v1"; +export type NoiseSuppressionModel = "near-field" | "far-field"; + export type StreamingTokenParams = { expires_in_seconds: number; max_session_duration_seconds?: number; @@ -139,9 +145,17 @@ export type StreamingForceEndpoint = { }; export type ErrorEvent = { + type: "Error"; + error_code?: number; error: string; }; +export type WarningEvent = { + type: "Warning"; + warning_code: number; + warning: string; +}; + export type LLMGatewayResponseEvent = { type: "LLMGatewayResponse"; turn_order: number; @@ -155,7 +169,8 @@ export type StreamingEventMessage = | SpeechStartedEvent | TerminationEvent | LLMGatewayResponseEvent - | ErrorEvent; + | ErrorEvent + | WarningEvent; export type StreamingOperationMessage = | StreamingUpdateConfiguration diff --git a/src/utils/errors/streaming.ts b/src/utils/errors/streaming.ts index 1a08667..6f0a234 100644 --- a/src/utils/errors/streaming.ts +++ b/src/utils/errors/streaming.ts @@ -15,12 +15,24 @@ const StreamingErrorType = { BadSchema: 4101, TooManyStreams: 4102, Reconnected: 4103, + ServerError: 3005, + InputValidationError: 3006, + AudioChunkDurationViolation: 3007, + MaxSessionDurationExceeded: 3008, + ConcurrencyLimitExceeded: 3009, } as const; type StreamingErrorTypeCodes = (typeof StreamingErrorType)[keyof typeof StreamingErrorType]; const StreamingErrorMessages: Record = { + [StreamingErrorType.ServerError]: "Server error", + [StreamingErrorType.InputValidationError]: "Input validation error", + [StreamingErrorType.AudioChunkDurationViolation]: + "Audio chunk duration violation", + [StreamingErrorType.MaxSessionDurationExceeded]: + "Session expired: maximum session duration exceeded", + [StreamingErrorType.ConcurrencyLimitExceeded]: "Too many concurrent sessions", [StreamingErrorType.BadSampleRate]: "Sample rate must be a positive integer", [StreamingErrorType.AuthFailed]: "Not Authorized", [StreamingErrorType.InsufficientFunds]: "Insufficient funds", diff --git a/tests/unit/streaming.test.ts b/tests/unit/streaming.test.ts index e01b99c..4c40204 100644 --- a/tests/unit/streaming.test.ts +++ b/tests/unit/streaming.test.ts @@ -98,6 +98,72 @@ describe("streaming", () => { await connect(rt, server); }); + it("should normalize deprecated minEndOfTurnSilenceWhenConfident to min_turn_silence in connection URL", async () => { + await cleanup(); + WS.clean(); + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&speech_model=universal-streaming-english&min_turn_silence=200`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + speechModel: "universal-streaming-english", + minEndOfTurnSilenceWhenConfident: 200, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + }); + + it("should prefer minTurnSilence when both are set", async () => { + await cleanup(); + WS.clean(); + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&speech_model=universal-streaming-english&min_turn_silence=500`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + speechModel: "universal-streaming-english", + minEndOfTurnSilenceWhenConfident: 200, + minTurnSilence: 500, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + }); + + it("should normalize deprecated min_end_of_turn_silence_when_confident in updateConfiguration", async () => { + rt.updateConfiguration({ min_end_of_turn_silence_when_confident: 200 }); + await expect(server).toReceiveMessage( + JSON.stringify({ + type: "UpdateConfiguration", + min_turn_silence: 200, + }), + ); + }); + + it("should include noise_suppression_model and noise_suppression_threshold in connection URL", async () => { + await cleanup(); + WS.clean(); + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&speech_model=universal-streaming-english&noise_suppression_model=near-field&noise_suppression_threshold=0.5`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + speechModel: "universal-streaming-english", + noiseSuppressionModel: "near-field", + noiseSuppressionThreshold: 0.5, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + }); + it("should include whisper-rt speech model in connection URL", async () => { await cleanup(); WS.clean(); diff --git a/tests/unit/transcript.test.ts b/tests/unit/transcript.test.ts index 73d8492..7ccdae4 100644 --- a/tests/unit/transcript.test.ts +++ b/tests/unit/transcript.test.ts @@ -568,4 +568,81 @@ describe("transcript", () => { expect(transcript.id).toBe(transcriptId); expect(transcript.metadata).toBeUndefined(); }); + + it("should include redact_pii_return_unredacted in the request body when set", async () => { + fetchMock.doMockOnceIf( + requestMatches({ url: "/v2/transcript", method: "POST" }), + JSON.stringify({ id: transcriptId, status: "queued" }), + ); + + await assembly.transcripts.submit({ + audio_url: remoteAudioURL, + redact_pii: true, + redact_pii_policies: ["person_name"], + redact_pii_return_unredacted: true, + }); + + const requestBody = JSON.parse(fetchMock.mock.calls[0][1]?.body as string); + expect(requestBody.redact_pii).toBe(true); + expect(requestBody.redact_pii_return_unredacted).toBe(true); + }); + + it("should expose unredacted_text, unredacted_words, and unredacted_utterances on the response", async () => { + const unredactedResponse = { + id: transcriptId, + status: "completed", + text: "Hi, my name is [PERSON_NAME].", + unredacted_text: "Hi, my name is Alice.", + words: [ + { text: "Hi,", start: 0, end: 100, confidence: 0.99 }, + { text: "my", start: 100, end: 200, confidence: 0.99 }, + { text: "name", start: 200, end: 300, confidence: 0.99 }, + { text: "is", start: 300, end: 400, confidence: 0.99 }, + { text: "[PERSON_NAME].", start: 400, end: 600, confidence: 0.99 }, + ], + unredacted_words: [ + { text: "Hi,", start: 0, end: 100, confidence: 0.99 }, + { text: "my", start: 100, end: 200, confidence: 0.99 }, + { text: "name", start: 200, end: 300, confidence: 0.99 }, + { text: "is", start: 300, end: 400, confidence: 0.99 }, + { text: "Alice.", start: 400, end: 600, confidence: 0.99 }, + ], + utterances: [ + { + text: "Hi, my name is [PERSON_NAME].", + start: 0, + end: 600, + confidence: 0.99, + speaker: "A", + words: [], + }, + ], + unredacted_utterances: [ + { + text: "Hi, my name is Alice.", + start: 0, + end: 600, + confidence: 0.99, + speaker: "A", + words: [], + }, + ], + }; + fetchMock.doMockOnceIf( + requestMatches({ url: `/v2/transcript/${transcriptId}`, method: "GET" }), + JSON.stringify(unredactedResponse), + ); + const transcript = await assembly.transcripts.get(transcriptId); + + expect(transcript.text).toBe("Hi, my name is [PERSON_NAME]."); + expect(transcript.unredacted_text).toBe("Hi, my name is Alice."); + expect(transcript.unredacted_words).toBeDefined(); + expect(transcript.unredacted_words!.length).toBe(5); + expect(transcript.unredacted_words![4].text).toBe("Alice."); + expect(transcript.unredacted_utterances).toBeDefined(); + expect(transcript.unredacted_utterances!.length).toBe(1); + expect(transcript.unredacted_utterances![0].text).toBe( + "Hi, my name is Alice.", + ); + }); });