diff --git a/config/_default/menus/api.en.yaml b/config/_default/menus/api.en.yaml index 1a51f78bd69..d04b4f2a654 100644 --- a/config/_default/menus/api.en.yaml +++ b/config/_default/menus/api.en.yaml @@ -12395,6 +12395,19 @@ menu: unstable: - v2 order: 18 + - name: Get annotated interactions by content IDs + url: '#get-annotated-interactions-by-content-ids' + identifier: llm-observability-get-annotated-interactions-by-content-ids + parent: llm-observability + generated: true + params: + versions: + - v2 + operationids: + - GetLLMObsAnnotatedInteractionsByTraceIDs + unstable: + - v2 + order: 40 - name: Delete LLM Observability data url: '#delete-llm-observability-data' identifier: llm-observability-delete-llm-observability-data diff --git a/content/en/api/v2/llm-observability/examples.json b/content/en/api/v2/llm-observability/examples.json index 487ef0de994..d9bac2e4380 100644 --- a/content/en/api/v2/llm-observability/examples.json +++ b/content/en/api/v2/llm-observability/examples.json @@ -586,6 +586,164 @@ "html": "
\n
\n
\n
\n

data [required]

\n
\n

object

\n

Data object for an LLM Observability data deletion request.

\n
\n
\n
\n
\n
\n

attributes [required]

\n
\n

object

\n

Attributes for an LLM Observability data deletion request.

\n
\n
\n
\n
\n
\n

delay

\n
\n

int64

\n

Optional delay in seconds before the deletion is executed.

\n
\n \n
\n
\n
\n
\n
\n

from [required]

\n
\n

int64

\n

Start of the deletion time range in milliseconds since Unix epoch.

\n
\n \n
\n
\n
\n
\n
\n

query [required]

\n
\n

object

\n

Query filters selecting the data to delete. Must include a query key with an @trace_id filter.

\n
\n
\n
\n
\n
\n

<any-key>

\n
\n

string

\n
\n
\n \n
\n
\n
\n
\n
\n
\n
\n

to [required]

\n
\n

int64

\n

End of the deletion time range in milliseconds since Unix epoch.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

type [required]

\n
\n

enum

\n

Resource type for an LLM Observability data deletion request. \nAllowed enum values: create_deletion_req

\n
\n \n
\n
\n
\n
" } }, + "GetLLMObsAnnotatedInteractionsByTraceIDs": { + "responses": { + "200": { + "json": { + "data": { + "attributes": { + "annotated_interactions": [ + { + "annotations": [ + { + "created_at": "2024-01-15T10:30:00Z", + "created_by": "00000000-0000-0000-0000-000000000002", + "id": "annotation-789", + "interaction_id": "interaction-456", + "label_values": { + "quality": "good" + }, + "modified_at": "2024-01-15T10:30:00Z", + "modified_by": "00000000-0000-0000-0000-000000000002" + } + ], + "content_id": "trace-abc-123", + "created_at": "2025-06-01T12:00:00Z", + "display_block": [ + { + "alt": "Example image", + "content": "## Triage Instructions", + "height": 240, + "interactionType": "trace", + "label": "Triage Instructions", + "level": "md", + "tileDef": { + "requests": [ + { + "queries": [ + { + "data_source": "metrics", + "name": "q", + "query": "avg:system.cpu.user{*}" + } + ], + "response_format": "timeseries", + "type": "line" + } + ], + "viz": "timeseries" + }, + "timeFrame": { + "end": 1705315800000, + "start": 1705312200000 + }, + "traceId": "69fcc2bb0000000003113989d83069ba", + "type": "markdown", + "url": "https://example.com/image.png" + } + ], + "id": "interaction-456", + "modified_at": "2025-06-01T12:00:00Z", + "queue_id": "queue-uuid-001", + "queue_name": "My Annotation Queue", + "type": "trace" + } + ], + "total_count": 1 + }, + "id": "trace-query", + "type": "annotated_interactions_by_trace" + } + }, + "html": "
\n
\n
\n
\n

data [required]

\n
\n

object

\n

Data object for the cross-queue annotated interactions response.

\n
\n
\n
\n
\n
\n

attributes [required]

\n
\n

object

\n

Attributes of the cross-queue annotated interactions response.

\n
\n
\n
\n
\n
\n

annotated_interactions [required]

\n
\n

[object]

\n

List of annotated interactions across all queues for the requested content IDs.

\n
\n
\n
\n
\n
\n

annotations [required]

\n
\n

[object]

\n

List of annotations for this interaction.

\n
\n
\n
\n
\n
\n

created_at [required]

\n
\n

date-time

\n

Timestamp when the annotation was created.

\n
\n \n
\n
\n
\n
\n
\n

created_by [required]

\n
\n

string

\n

Identifier of the user who created the annotation.

\n
\n \n
\n
\n
\n
\n
\n

id [required]

\n
\n

string

\n

Unique identifier of the annotation.

\n
\n \n
\n
\n
\n
\n
\n

interaction_id [required]

\n
\n

string

\n

Identifier of the interaction this annotation belongs to.

\n
\n \n
\n
\n
\n
\n
\n

label_values [required]

\n
\n

object

\n

The label values for this annotation.

\n
\n \n
\n
\n
\n
\n
\n

modified_at [required]

\n
\n

date-time

\n

Timestamp when the annotation was last modified.

\n
\n \n
\n
\n
\n
\n
\n

modified_by [required]

\n
\n

string

\n

Identifier of the user who last modified the annotation.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

content_id [required]

\n
\n

string

\n

Upstream entity identifier (trace ID, session ID, or deterministic display_block ID).

\n
\n \n
\n
\n
\n
\n
\n

created_at [required]

\n
\n

date-time

\n

Timestamp when the interaction was added to the queue.

\n
\n \n
\n
\n
\n
\n
\n

display_block

\n
\n

[object]

\n

List of content blocks that make up a display_block interaction.\nMust contain at least one block.

\n
\n
\n
\n
\n
\n

alt

\n
\n

string

\n

Alternative text for an image block.

\n
\n \n
\n
\n
\n
\n
\n

content

\n
\n

\n

Block payload. A string for markdown, header, and text; an\narbitrary JSON value (object, array, or scalar) for json. Omitted\nfor image, widget, and llmobs_trace.

\n
\n \n
\n
\n
\n
\n
\n

height

\n
\n

int64

\n

Optional rendered height. Must be positive when set.

\n
\n \n
\n
\n
\n
\n
\n

interactionType

\n
\n

enum

\n

Upstream interaction type referenced by an llmobs_trace block.\nRestricted to trace or experiment_trace. \nAllowed enum values: trace,experiment_trace

\n
\n \n
\n
\n
\n
\n
\n

label

\n
\n

string

\n

Optional label rendered alongside the block.

\n
\n \n
\n
\n
\n
\n
\n

level

\n
\n

enum

\n

Visual size for a header block. \nAllowed enum values: sm,md,lg,xl

\n
\n \n
\n
\n
\n
\n
\n

tileDef

\n
\n

\n

Tile definition for a widget block. Required for widget. The\nschema is owned by the frontend renderer.

\n
\n \n
\n
\n
\n
\n
\n

timeFrame

\n
\n

object

\n

Unix-millis time range used by chart blocks.

\n
\n
\n
\n
\n
\n

end [required]

\n
\n

int64

\n

End of the range, in Unix milliseconds.

\n
\n \n
\n
\n
\n
\n
\n

start [required]

\n
\n

int64

\n

Start of the range, in Unix milliseconds.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

traceId

\n
\n

string

\n

Trace identifier. Required for llmobs_trace blocks.

\n
\n \n
\n
\n
\n
\n
\n

type [required]

\n
\n

enum

\n

Discriminator for a single display_block content block. Adding a\nvariant requires coordinated changes in the frontend renderer. \nAllowed enum values: markdown,header,text,json,image,widget,llmobs_trace

\n
\n \n
\n
\n
\n
\n
\n

url

\n
\n

string

\n

URL of the image. Required for image blocks.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

id [required]

\n
\n

string

\n

Unique identifier of the interaction.

\n
\n \n
\n
\n
\n
\n
\n

modified_at [required]

\n
\n

date-time

\n

Timestamp when the interaction was last updated.

\n
\n \n
\n
\n
\n
\n
\n

queue_id [required]

\n
\n

string

\n

Identifier of the annotation queue this interaction belongs to.

\n
\n \n
\n
\n
\n
\n
\n

queue_name [required]

\n
\n

string

\n

Name of the annotation queue this interaction belongs to.

\n
\n \n
\n
\n
\n
\n
\n

type [required]

\n
\n

enum

\n

Type of an annotated interaction. \nAllowed enum values: trace,experiment_trace,session,display_block

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

total_count [required]

\n
\n

int32

\n

Total number of annotated interactions matching the query.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

id [required]

\n
\n

string

\n

Opaque identifier for the response object.

\n
\n \n
\n
\n
\n
\n
\n

type [required]

\n
\n

enum

\n

Resource type for cross-queue annotated interactions lookup. \nAllowed enum values: annotated_interactions_by_trace

\n
\n \n
\n
\n
\n
" + }, + "400": { + "json": { + "errors": [ + { + "detail": "Missing required attribute in body", + "meta": {}, + "source": { + "header": "Authorization", + "parameter": "limit", + "pointer": "/data/attributes/title" + }, + "status": "400", + "title": "Bad Request" + } + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[object]

\n

A list of errors.

\n
\n
\n
\n
\n
\n

detail

\n
\n

string

\n

A human-readable explanation specific to this occurrence of the error.

\n
\n \n
\n
\n
\n
\n
\n

meta

\n
\n

object

\n

Non-standard meta-information about the error

\n
\n \n
\n
\n
\n
\n
\n

source

\n
\n

object

\n

References to the source of the error.

\n
\n
\n
\n
\n
\n

header

\n
\n

string

\n

A string indicating the name of a single request header which caused the error.

\n
\n \n
\n
\n
\n
\n
\n

parameter

\n
\n

string

\n

A string indicating which URI query parameter caused the error.

\n
\n \n
\n
\n
\n
\n
\n

pointer

\n
\n

string

\n

A JSON pointer to the value in the request document that caused the error.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

status

\n
\n

string

\n

Status code of the response.

\n
\n \n
\n
\n
\n
\n
\n

title

\n
\n

string

\n

Short human-readable summary of the error.

\n
\n \n
\n
\n
\n
" + }, + "401": { + "json": { + "errors": [ + { + "detail": "Missing required attribute in body", + "meta": {}, + "source": { + "header": "Authorization", + "parameter": "limit", + "pointer": "/data/attributes/title" + }, + "status": "400", + "title": "Bad Request" + } + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[object]

\n

A list of errors.

\n
\n
\n
\n
\n
\n

detail

\n
\n

string

\n

A human-readable explanation specific to this occurrence of the error.

\n
\n \n
\n
\n
\n
\n
\n

meta

\n
\n

object

\n

Non-standard meta-information about the error

\n
\n \n
\n
\n
\n
\n
\n

source

\n
\n

object

\n

References to the source of the error.

\n
\n
\n
\n
\n
\n

header

\n
\n

string

\n

A string indicating the name of a single request header which caused the error.

\n
\n \n
\n
\n
\n
\n
\n

parameter

\n
\n

string

\n

A string indicating which URI query parameter caused the error.

\n
\n \n
\n
\n
\n
\n
\n

pointer

\n
\n

string

\n

A JSON pointer to the value in the request document that caused the error.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

status

\n
\n

string

\n

Status code of the response.

\n
\n \n
\n
\n
\n
\n
\n

title

\n
\n

string

\n

Short human-readable summary of the error.

\n
\n \n
\n
\n
\n
" + }, + "403": { + "json": { + "errors": [ + { + "detail": "Missing required attribute in body", + "meta": {}, + "source": { + "header": "Authorization", + "parameter": "limit", + "pointer": "/data/attributes/title" + }, + "status": "400", + "title": "Bad Request" + } + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[object]

\n

A list of errors.

\n
\n
\n
\n
\n
\n

detail

\n
\n

string

\n

A human-readable explanation specific to this occurrence of the error.

\n
\n \n
\n
\n
\n
\n
\n

meta

\n
\n

object

\n

Non-standard meta-information about the error

\n
\n \n
\n
\n
\n
\n
\n

source

\n
\n

object

\n

References to the source of the error.

\n
\n
\n
\n
\n
\n

header

\n
\n

string

\n

A string indicating the name of a single request header which caused the error.

\n
\n \n
\n
\n
\n
\n
\n

parameter

\n
\n

string

\n

A string indicating which URI query parameter caused the error.

\n
\n \n
\n
\n
\n
\n
\n

pointer

\n
\n

string

\n

A JSON pointer to the value in the request document that caused the error.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

status

\n
\n

string

\n

Status code of the response.

\n
\n \n
\n
\n
\n
\n
\n

title

\n
\n

string

\n

Short human-readable summary of the error.

\n
\n \n
\n
\n
\n
" + }, + "429": { + "json": { + "errors": [ + "Bad Request" + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[string]

\n

A list of errors.

\n
\n \n
\n
" + }, + "500": { + "json": { + "errors": [ + { + "detail": "Missing required attribute in body", + "meta": {}, + "source": { + "header": "Authorization", + "parameter": "limit", + "pointer": "/data/attributes/title" + }, + "status": "400", + "title": "Bad Request" + } + ] + }, + "html": "
\n
\n
\n
\n

errors [required]

\n
\n

[object]

\n

A list of errors.

\n
\n
\n
\n
\n
\n

detail

\n
\n

string

\n

A human-readable explanation specific to this occurrence of the error.

\n
\n \n
\n
\n
\n
\n
\n

meta

\n
\n

object

\n

Non-standard meta-information about the error

\n
\n \n
\n
\n
\n
\n
\n

source

\n
\n

object

\n

References to the source of the error.

\n
\n
\n
\n
\n
\n

header

\n
\n

string

\n

A string indicating the name of a single request header which caused the error.

\n
\n \n
\n
\n
\n
\n
\n

parameter

\n
\n

string

\n

A string indicating which URI query parameter caused the error.

\n
\n \n
\n
\n
\n
\n
\n

pointer

\n
\n

string

\n

A JSON pointer to the value in the request document that caused the error.

\n
\n \n
\n
\n
\n
\n
\n
\n
\n

status

\n
\n

string

\n

Status code of the response.

\n
\n \n
\n
\n
\n
\n
\n

title

\n
\n

string

\n

Short human-readable summary of the error.

\n
\n \n
\n
\n
\n
" + } + }, + "request": { + "json_curl": {}, + "json": {}, + "html": "" + } + }, "ListLLMObsAnnotationQueues": { "responses": { "200": { diff --git a/data/api/v2/full_spec.yaml b/data/api/v2/full_spec.yaml index 817ace56ae4..71eb184e537 100644 --- a/data/api/v2/full_spec.yaml +++ b/data/api/v2/full_spec.yaml @@ -42108,11 +42108,109 @@ components: format: int64 type: integer type: object + LLMObsAnnotatedInteractionByTraceItem: + description: An annotated interaction returned by the cross-queue lookup, including the source queue metadata. + properties: + annotations: + description: List of annotations for this interaction. + items: + $ref: "#/components/schemas/LLMObsAnnotationItem" + type: array + content_id: + description: Upstream entity identifier (trace ID, session ID, or deterministic display_block ID). + example: "trace-abc-123" + type: string + created_at: + description: Timestamp when the interaction was added to the queue. + example: "2025-06-01T12:00:00Z" + format: date-time + type: string + display_block: + $ref: "#/components/schemas/LLMObsContentBlocks" + id: + description: Unique identifier of the interaction. + example: "interaction-456" + type: string + modified_at: + description: Timestamp when the interaction was last updated. + example: "2025-06-01T12:00:00Z" + format: date-time + type: string + queue_id: + description: Identifier of the annotation queue this interaction belongs to. + example: "queue-uuid-001" + type: string + queue_name: + description: Name of the annotation queue this interaction belongs to. + example: "My Annotation Queue" + type: string + type: + $ref: "#/components/schemas/LLMObsAnyInteractionType" + required: + - id + - type + - content_id + - created_at + - modified_at + - queue_id + - queue_name + - annotations + type: object LLMObsAnnotatedInteractionItem: description: An interaction with its associated annotations. oneOf: - $ref: "#/components/schemas/LLMObsTraceAnnotatedInteractionItem" - $ref: "#/components/schemas/LLMObsDisplayBlockAnnotatedInteractionItem" + LLMObsAnnotatedInteractionsByTraceDataAttributesResponse: + description: Attributes of the cross-queue annotated interactions response. + properties: + annotated_interactions: + description: List of annotated interactions across all queues for the requested content IDs. + items: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionByTraceItem" + type: array + total_count: + description: Total number of annotated interactions matching the query. + example: 1 + format: int32 + maximum: 2147483647 + type: integer + required: + - annotated_interactions + - total_count + type: object + LLMObsAnnotatedInteractionsByTraceDataResponse: + description: Data object for the cross-queue annotated interactions response. + properties: + attributes: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsByTraceDataAttributesResponse" + id: + description: Opaque identifier for the response object. + example: "trace-query" + type: string + type: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsByTraceType" + required: + - id + - type + - attributes + type: object + LLMObsAnnotatedInteractionsByTraceResponse: + description: Response containing annotated interactions across all queues for the requested content IDs. + properties: + data: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsByTraceDataResponse" + required: + - data + type: object + LLMObsAnnotatedInteractionsByTraceType: + description: Resource type for cross-queue annotated interactions lookup. + enum: + - annotated_interactions_by_trace + example: annotated_interactions_by_trace + type: string + x-enum-varnames: + - ANNOTATED_INTERACTIONS_BY_TRACE LLMObsAnnotatedInteractionsDataAttributesResponse: description: Attributes containing the list of annotated interactions. properties: @@ -42582,6 +42680,20 @@ components: - ENABLED - DISABLED - ADAPTIVE + LLMObsAnyInteractionType: + description: Type of an annotated interaction. + enum: + - trace + - experiment_trace + - session + - display_block + example: trace + type: string + x-enum-varnames: + - TRACE + - EXPERIMENT_TRACE + - SESSION + - DISPLAY_BLOCK LLMObsAzureOpenAIMetadata: description: Azure OpenAI-specific metadata for an integration account or inference request. properties: @@ -123605,6 +123717,105 @@ paths: x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/annotated-interactions: + get: + description: Returns annotated interactions across all annotation queues for the given content IDs. Results include queue metadata (ID and name) for each interaction. + operationId: GetLLMObsAnnotatedInteractionsByTraceIDs + parameters: + - description: One or more content IDs to retrieve annotated interactions for. At least one is required. + in: query + name: contentIds + required: true + schema: + items: + type: string + type: array + - description: Pagination offset. Must be >= 0. Defaults to 0. + in: query + name: offset + schema: + default: 0 + format: int32 + maximum: 2147483647 + minimum: 0 + type: integer + - description: Maximum number of results to return. Must be > 0. Defaults to 100. + in: query + name: limit + schema: + default: 100 + format: int32 + maximum: 2147483647 + minimum: 1 + type: integer + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + annotated_interactions: + - annotations: + - created_at: "0001-01-01T00:00:00Z" + created_by: "00000000-0000-0000-0000-000000000002" + id: annotation-789 + interaction_id: interaction-456 + label_values: + quality: good + modified_at: "0001-01-01T00:00:00Z" + modified_by: "00000000-0000-0000-0000-000000000002" + content_id: trace-abc-123 + created_at: "2025-06-01T12:00:00Z" + id: interaction-456 + modified_at: "2025-06-01T12:00:00Z" + queue_id: queue-uuid-001 + queue_name: My Annotation Queue + type: trace + total_count: 1 + id: trace-query + type: annotated_interactions_by_trace + schema: + $ref: "#/components/schemas/LLMObsAnnotatedInteractionsByTraceResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Get annotated interactions by content IDs + tags: + - LLM Observability + x-menu-order: 40 + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). /api/v2/llm-obs/v1/annotation-queues: get: description: |- diff --git a/data/api/v2/translate_actions.json b/data/api/v2/translate_actions.json index f75d4f6e64c..929cb10a8db 100644 --- a/data/api/v2/translate_actions.json +++ b/data/api/v2/translate_actions.json @@ -2531,6 +2531,10 @@ "request_description": "Data deletion request payload.", "request_schema_description": "Request to delete LLM Observability data." }, + "GetLLMObsAnnotatedInteractionsByTraceIDs": { + "description": "Returns annotated interactions across all annotation queues for the given content IDs. Results include queue metadata (ID and name) for each interaction.", + "summary": "Get annotated interactions by content IDs" + }, "ListLLMObsAnnotationQueues": { "description": "List annotation queues. Optionally filter by project ID or queue IDs. These parameters are mutually exclusive.\nIf neither is provided, all queues in the organization are returned.", "summary": "List LLM Observability annotation queues"