From 6c5973326f279f0c08ab1ff3a227790c3d069387 Mon Sep 17 00:00:00 2001 From: Arseny Boykov Date: Wed, 24 Jun 2026 00:24:25 +0400 Subject: [PATCH 1/5] proto(agent-sim): add AudioMetrics to SimulationRun.Job Carries audio-mode pipeline metrics aggregated per call: Tier-0 end-to-end response latency (p50/p95) off the waveform, and Tier-1 means (WER, LLM TTFT, STT delay, TTS TTFB) plus barge-in count. -1 marks an unavailable datum (Tier-1 needs the agent backchannel). Co-Authored-By: Claude Opus 4.8 --- livekit/livekit_agent_simulation.pb.go | 304 ++++++++++++++++------- protobufs/livekit_agent_simulation.proto | 20 ++ 2 files changed, 237 insertions(+), 87 deletions(-) diff --git a/livekit/livekit_agent_simulation.pb.go b/livekit/livekit_agent_simulation.pb.go index e4cee95dd..4e71d9646 100644 --- a/livekit/livekit_agent_simulation.pb.go +++ b/livekit/livekit_agent_simulation.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v7.35.1 +// protoc v7.35.0 // source: livekit_agent_simulation.proto package livekit @@ -750,8 +750,12 @@ type SimulationRun_Job struct { EndedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` RoomId string `protobuf:"bytes,12,opt,name=room_id,json=roomId,proto3" json:"room_id,omitempty"` Usage *SimulationRun_Job_Usage `protobuf:"bytes,13,opt,name=usage,proto3" json:"usage,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Audio-mode pipeline metrics aggregated over the call's measured turns. + // Only populated for audio jobs. A -1 field means the datum was + // unavailable (Tier-1 metrics require the agent's backchannel). + AudioMetrics *SimulationRun_Job_AudioMetrics `protobuf:"bytes,14,opt,name=audio_metrics,json=audioMetrics,proto3" json:"audio_metrics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_Job) Reset() { @@ -868,6 +872,13 @@ func (x *SimulationRun_Job) GetUsage() *SimulationRun_Job_Usage { return nil } +func (x *SimulationRun_Job) GetAudioMetrics() *SimulationRun_Job_AudioMetrics { + if x != nil { + return x.AudioMetrics + } + return nil +} + type SimulationRun_Create struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1152,6 +1163,111 @@ func (x *SimulationRun_Job_Usage) GetAudioTurnsCount() int32 { return 0 } +type SimulationRun_Job_AudioMetrics struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Tier 0 — true end-to-end response latency off the waveform (our + // playout end -> the agent's speech onset). Always present for audio. + E2ELatencyP50S float32 `protobuf:"fixed32,1,opt,name=e2e_latency_p50_s,json=e2eLatencyP50S,proto3" json:"e2e_latency_p50_s,omitempty"` + E2ELatencyP95S float32 `protobuf:"fixed32,2,opt,name=e2e_latency_p95_s,json=e2eLatencyP95S,proto3" json:"e2e_latency_p95_s,omitempty"` + // Tier 1 — mean over turns; -1 when the backchannel is absent. + Wer float32 `protobuf:"fixed32,3,opt,name=wer,proto3" json:"wer,omitempty"` // normalized word error rate of the agent's STT of us + LlmTtftS float32 `protobuf:"fixed32,4,opt,name=llm_ttft_s,json=llmTtftS,proto3" json:"llm_ttft_s,omitempty"` + SttDelayS float32 `protobuf:"fixed32,5,opt,name=stt_delay_s,json=sttDelayS,proto3" json:"stt_delay_s,omitempty"` + TtsTtfbS float32 `protobuf:"fixed32,6,opt,name=tts_ttfb_s,json=ttsTtfbS,proto3" json:"tts_ttfb_s,omitempty"` + // Call-level barge-in count. + Interruptions int32 `protobuf:"varint,7,opt,name=interruptions,proto3" json:"interruptions,omitempty"` + // Turns that contributed measurements (context for the aggregates). + SampleCount int32 `protobuf:"varint,8,opt,name=sample_count,json=sampleCount,proto3" json:"sample_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SimulationRun_Job_AudioMetrics) Reset() { + *x = SimulationRun_Job_AudioMetrics{} + mi := &file_livekit_agent_simulation_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SimulationRun_Job_AudioMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimulationRun_Job_AudioMetrics) ProtoMessage() {} + +func (x *SimulationRun_Job_AudioMetrics) ProtoReflect() protoreflect.Message { + mi := &file_livekit_agent_simulation_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimulationRun_Job_AudioMetrics.ProtoReflect.Descriptor instead. +func (*SimulationRun_Job_AudioMetrics) Descriptor() ([]byte, []int) { + return file_livekit_agent_simulation_proto_rawDescGZIP(), []int{1, 0, 1} +} + +func (x *SimulationRun_Job_AudioMetrics) GetE2ELatencyP50S() float32 { + if x != nil { + return x.E2ELatencyP50S + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetE2ELatencyP95S() float32 { + if x != nil { + return x.E2ELatencyP95S + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetWer() float32 { + if x != nil { + return x.Wer + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetLlmTtftS() float32 { + if x != nil { + return x.LlmTtftS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetSttDelayS() float32 { + if x != nil { + return x.SttDelayS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsTtfbS() float32 { + if x != nil { + return x.TtsTtfbS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetInterruptions() int32 { + if x != nil { + return x.Interruptions + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetSampleCount() int32 { + if x != nil { + return x.SampleCount + } + return 0 +} + type SimulationRun_Create_Request struct { state protoimpl.MessageState `protogen:"open.v1"` ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` @@ -1172,7 +1288,7 @@ type SimulationRun_Create_Request struct { func (x *SimulationRun_Create_Request) Reset() { *x = SimulationRun_Create_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[15] + mi := &file_livekit_agent_simulation_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1184,7 +1300,7 @@ func (x *SimulationRun_Create_Request) String() string { func (*SimulationRun_Create_Request) ProtoMessage() {} func (x *SimulationRun_Create_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[15] + mi := &file_livekit_agent_simulation_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1259,7 +1375,7 @@ type SimulationRun_Create_Response struct { func (x *SimulationRun_Create_Response) Reset() { *x = SimulationRun_Create_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[16] + mi := &file_livekit_agent_simulation_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1271,7 +1387,7 @@ func (x *SimulationRun_Create_Response) String() string { func (*SimulationRun_Create_Response) ProtoMessage() {} func (x *SimulationRun_Create_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[16] + mi := &file_livekit_agent_simulation_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1312,7 +1428,7 @@ type SimulationRun_ConfirmSourceUpload_Request struct { func (x *SimulationRun_ConfirmSourceUpload_Request) Reset() { *x = SimulationRun_ConfirmSourceUpload_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[17] + mi := &file_livekit_agent_simulation_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1324,7 +1440,7 @@ func (x *SimulationRun_ConfirmSourceUpload_Request) String() string { func (*SimulationRun_ConfirmSourceUpload_Request) ProtoMessage() {} func (x *SimulationRun_ConfirmSourceUpload_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[17] + mi := &file_livekit_agent_simulation_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1369,7 +1485,7 @@ type SimulationRun_ConfirmSourceUpload_Response struct { func (x *SimulationRun_ConfirmSourceUpload_Response) Reset() { *x = SimulationRun_ConfirmSourceUpload_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[18] + mi := &file_livekit_agent_simulation_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1381,7 +1497,7 @@ func (x *SimulationRun_ConfirmSourceUpload_Response) String() string { func (*SimulationRun_ConfirmSourceUpload_Response) ProtoMessage() {} func (x *SimulationRun_ConfirmSourceUpload_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[18] + mi := &file_livekit_agent_simulation_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1407,7 +1523,7 @@ type SimulationRun_Get_Request struct { func (x *SimulationRun_Get_Request) Reset() { *x = SimulationRun_Get_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[19] + mi := &file_livekit_agent_simulation_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1419,7 +1535,7 @@ func (x *SimulationRun_Get_Request) String() string { func (*SimulationRun_Get_Request) ProtoMessage() {} func (x *SimulationRun_Get_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[19] + mi := &file_livekit_agent_simulation_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1458,7 +1574,7 @@ type SimulationRun_Get_Response struct { func (x *SimulationRun_Get_Response) Reset() { *x = SimulationRun_Get_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[20] + mi := &file_livekit_agent_simulation_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1470,7 +1586,7 @@ func (x *SimulationRun_Get_Response) String() string { func (*SimulationRun_Get_Response) ProtoMessage() {} func (x *SimulationRun_Get_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[20] + mi := &file_livekit_agent_simulation_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1504,7 +1620,7 @@ type SimulationRun_List_Request struct { func (x *SimulationRun_List_Request) Reset() { *x = SimulationRun_List_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[21] + mi := &file_livekit_agent_simulation_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1516,7 +1632,7 @@ func (x *SimulationRun_List_Request) String() string { func (*SimulationRun_List_Request) ProtoMessage() {} func (x *SimulationRun_List_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[21] + mi := &file_livekit_agent_simulation_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1563,7 +1679,7 @@ type SimulationRun_List_Response struct { func (x *SimulationRun_List_Response) Reset() { *x = SimulationRun_List_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[22] + mi := &file_livekit_agent_simulation_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1575,7 +1691,7 @@ func (x *SimulationRun_List_Response) String() string { func (*SimulationRun_List_Response) ProtoMessage() {} func (x *SimulationRun_List_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[22] + mi := &file_livekit_agent_simulation_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1615,7 +1731,7 @@ type SimulationRun_Cancel_Request struct { func (x *SimulationRun_Cancel_Request) Reset() { *x = SimulationRun_Cancel_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[23] + mi := &file_livekit_agent_simulation_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1627,7 +1743,7 @@ func (x *SimulationRun_Cancel_Request) String() string { func (*SimulationRun_Cancel_Request) ProtoMessage() {} func (x *SimulationRun_Cancel_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[23] + mi := &file_livekit_agent_simulation_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1665,7 +1781,7 @@ type SimulationRun_Cancel_Response struct { func (x *SimulationRun_Cancel_Response) Reset() { *x = SimulationRun_Cancel_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[24] + mi := &file_livekit_agent_simulation_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1677,7 +1793,7 @@ func (x *SimulationRun_Cancel_Response) String() string { func (*SimulationRun_Cancel_Response) ProtoMessage() {} func (x *SimulationRun_Cancel_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[24] + mi := &file_livekit_agent_simulation_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1701,7 +1817,7 @@ type Scenario_CreateFromSession struct { func (x *Scenario_CreateFromSession) Reset() { *x = Scenario_CreateFromSession{} - mi := &file_livekit_agent_simulation_proto_msgTypes[25] + mi := &file_livekit_agent_simulation_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1713,7 +1829,7 @@ func (x *Scenario_CreateFromSession) String() string { func (*Scenario_CreateFromSession) ProtoMessage() {} func (x *Scenario_CreateFromSession) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[25] + mi := &file_livekit_agent_simulation_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1740,7 +1856,7 @@ type Scenario_CreateFromSession_Request struct { func (x *Scenario_CreateFromSession_Request) Reset() { *x = Scenario_CreateFromSession_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[27] + mi := &file_livekit_agent_simulation_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1752,7 +1868,7 @@ func (x *Scenario_CreateFromSession_Request) String() string { func (*Scenario_CreateFromSession_Request) ProtoMessage() {} func (x *Scenario_CreateFromSession_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[27] + mi := &file_livekit_agent_simulation_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1798,7 +1914,7 @@ type Scenario_CreateFromSession_Response struct { func (x *Scenario_CreateFromSession_Response) Reset() { *x = Scenario_CreateFromSession_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[28] + mi := &file_livekit_agent_simulation_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1810,7 +1926,7 @@ func (x *Scenario_CreateFromSession_Response) String() string { func (*Scenario_CreateFromSession_Response) ProtoMessage() {} func (x *Scenario_CreateFromSession_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[28] + mi := &file_livekit_agent_simulation_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1855,7 +1971,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\n" + "suggestion\x18\x02 \x01(\tR\n" + "suggestion\x12\x14\n" + - "\x05label\x18\x03 \x01(\tR\x05label\"\x80\x17\n" + + "\x05label\x18\x03 \x01(\tR\x05label\"\xec\x19\n" + "\rSimulationRun\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1d\n" + "\n" + @@ -1878,7 +1994,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x0fnum_simulations\x18\x0f \x01(\x05R\x0enumSimulations\x122\n" + "\x05usage\x18\x10 \x01(\v2\x1c.livekit.SimulationRun.UsageR\x05usage\x12 \n" + "\vconcurrency\x18\x11 \x01(\x05R\vconcurrency\x12+\n" + - "\x04mode\x18\x12 \x01(\x0e2\x17.livekit.SimulationModeR\x04mode\x1a\x99\x05\n" + + "\x04mode\x18\x12 \x01(\x0e2\x17.livekit.SimulationModeR\x04mode\x1a\x85\b\n" + "\x03Job\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x129\n" + "\x06status\x18\x02 \x01(\x0e2!.livekit.SimulationRun.Job.StatusR\x06status\x12\"\n" + @@ -1893,10 +2009,22 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + " \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + "\bended_at\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\x12\x17\n" + "\aroom_id\x18\f \x01(\tR\x06roomId\x126\n" + - "\x05usage\x18\r \x01(\v2 .livekit.SimulationRun.Job.UsageR\x05usage\x1a]\n" + + "\x05usage\x18\r \x01(\v2 .livekit.SimulationRun.Job.UsageR\x05usage\x12L\n" + + "\raudio_metrics\x18\x0e \x01(\v2'.livekit.SimulationRun.Job.AudioMetricsR\faudioMetrics\x1a]\n" + "\x05Usage\x12(\n" + "\x10text_turns_count\x18\x01 \x01(\x05R\x0etextTurnsCount\x12*\n" + - "\x11audio_turns_count\x18\x02 \x01(\x05R\x0faudioTurnsCount\"o\n" + + "\x11audio_turns_count\x18\x02 \x01(\x05R\x0faudioTurnsCount\x1a\x9b\x02\n" + + "\fAudioMetrics\x12)\n" + + "\x11e2e_latency_p50_s\x18\x01 \x01(\x02R\x0ee2eLatencyP50S\x12)\n" + + "\x11e2e_latency_p95_s\x18\x02 \x01(\x02R\x0ee2eLatencyP95S\x12\x10\n" + + "\x03wer\x18\x03 \x01(\x02R\x03wer\x12\x1c\n" + + "\n" + + "llm_ttft_s\x18\x04 \x01(\x02R\bllmTtftS\x12\x1e\n" + + "\vstt_delay_s\x18\x05 \x01(\x02R\tsttDelayS\x12\x1c\n" + + "\n" + + "tts_ttfb_s\x18\x06 \x01(\x02R\bttsTtfbS\x12$\n" + + "\rinterruptions\x18\a \x01(\x05R\rinterruptions\x12!\n" + + "\fsample_count\x18\b \x01(\x05R\vsampleCount\"o\n" + "\x06Status\x12\x12\n" + "\x0eSTATUS_PENDING\x10\x00\x12\x12\n" + "\x0eSTATUS_RUNNING\x10\x01\x12\x14\n" + @@ -2015,7 +2143,7 @@ func file_livekit_agent_simulation_proto_rawDescGZIP() []byte { } var file_livekit_agent_simulation_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_livekit_agent_simulation_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_livekit_agent_simulation_proto_msgTypes = make([]protoimpl.MessageInfo, 30) var file_livekit_agent_simulation_proto_goTypes = []any{ (SimulationMode)(0), // 0: livekit.SimulationMode (SimulationRun_Status)(0), // 1: livekit.SimulationRun.Status @@ -2035,71 +2163,73 @@ var file_livekit_agent_simulation_proto_goTypes = []any{ (*SimulationRun_Cancel)(nil), // 15: livekit.SimulationRun.Cancel (*SimulationRun_Usage)(nil), // 16: livekit.SimulationRun.Usage (*SimulationRun_Job_Usage)(nil), // 17: livekit.SimulationRun.Job.Usage - (*SimulationRun_Create_Request)(nil), // 18: livekit.SimulationRun.Create.Request - (*SimulationRun_Create_Response)(nil), // 19: livekit.SimulationRun.Create.Response - (*SimulationRun_ConfirmSourceUpload_Request)(nil), // 20: livekit.SimulationRun.ConfirmSourceUpload.Request - (*SimulationRun_ConfirmSourceUpload_Response)(nil), // 21: livekit.SimulationRun.ConfirmSourceUpload.Response - (*SimulationRun_Get_Request)(nil), // 22: livekit.SimulationRun.Get.Request - (*SimulationRun_Get_Response)(nil), // 23: livekit.SimulationRun.Get.Response - (*SimulationRun_List_Request)(nil), // 24: livekit.SimulationRun.List.Request - (*SimulationRun_List_Response)(nil), // 25: livekit.SimulationRun.List.Response - (*SimulationRun_Cancel_Request)(nil), // 26: livekit.SimulationRun.Cancel.Request - (*SimulationRun_Cancel_Response)(nil), // 27: livekit.SimulationRun.Cancel.Response - (*Scenario_CreateFromSession)(nil), // 28: livekit.Scenario.CreateFromSession - nil, // 29: livekit.Scenario.TagsEntry - (*Scenario_CreateFromSession_Request)(nil), // 30: livekit.Scenario.CreateFromSession.Request - (*Scenario_CreateFromSession_Response)(nil), // 31: livekit.Scenario.CreateFromSession.Response - (*timestamppb.Timestamp)(nil), // 32: google.protobuf.Timestamp - (*agent.ChatContext)(nil), // 33: livekit.agent.ChatContext - (*PresignedPostRequest)(nil), // 34: livekit.PresignedPostRequest - (*TokenPagination)(nil), // 35: livekit.TokenPagination + (*SimulationRun_Job_AudioMetrics)(nil), // 18: livekit.SimulationRun.Job.AudioMetrics + (*SimulationRun_Create_Request)(nil), // 19: livekit.SimulationRun.Create.Request + (*SimulationRun_Create_Response)(nil), // 20: livekit.SimulationRun.Create.Response + (*SimulationRun_ConfirmSourceUpload_Request)(nil), // 21: livekit.SimulationRun.ConfirmSourceUpload.Request + (*SimulationRun_ConfirmSourceUpload_Response)(nil), // 22: livekit.SimulationRun.ConfirmSourceUpload.Response + (*SimulationRun_Get_Request)(nil), // 23: livekit.SimulationRun.Get.Request + (*SimulationRun_Get_Response)(nil), // 24: livekit.SimulationRun.Get.Response + (*SimulationRun_List_Request)(nil), // 25: livekit.SimulationRun.List.Request + (*SimulationRun_List_Response)(nil), // 26: livekit.SimulationRun.List.Response + (*SimulationRun_Cancel_Request)(nil), // 27: livekit.SimulationRun.Cancel.Request + (*SimulationRun_Cancel_Response)(nil), // 28: livekit.SimulationRun.Cancel.Response + (*Scenario_CreateFromSession)(nil), // 29: livekit.Scenario.CreateFromSession + nil, // 30: livekit.Scenario.TagsEntry + (*Scenario_CreateFromSession_Request)(nil), // 31: livekit.Scenario.CreateFromSession.Request + (*Scenario_CreateFromSession_Response)(nil), // 32: livekit.Scenario.CreateFromSession.Response + (*timestamppb.Timestamp)(nil), // 33: google.protobuf.Timestamp + (*agent.ChatContext)(nil), // 34: livekit.agent.ChatContext + (*PresignedPostRequest)(nil), // 35: livekit.PresignedPostRequest + (*TokenPagination)(nil), // 36: livekit.TokenPagination } var file_livekit_agent_simulation_proto_depIdxs = []int32{ 9, // 0: livekit.SimulationRunSummary.issues:type_name -> livekit.SimulationRunSummary.Issue 8, // 1: livekit.SimulationRunSummary.chat_history:type_name -> livekit.SimulationRunSummary.ChatHistoryEntry 1, // 2: livekit.SimulationRun.status:type_name -> livekit.SimulationRun.Status - 32, // 3: livekit.SimulationRun.created_at:type_name -> google.protobuf.Timestamp + 33, // 3: livekit.SimulationRun.created_at:type_name -> google.protobuf.Timestamp 10, // 4: livekit.SimulationRun.jobs:type_name -> livekit.SimulationRun.Job 3, // 5: livekit.SimulationRun.summary:type_name -> livekit.SimulationRunSummary 6, // 6: livekit.SimulationRun.scenario_group:type_name -> livekit.ScenarioGroup - 32, // 7: livekit.SimulationRun.ended_at:type_name -> google.protobuf.Timestamp + 33, // 7: livekit.SimulationRun.ended_at:type_name -> google.protobuf.Timestamp 16, // 8: livekit.SimulationRun.usage:type_name -> livekit.SimulationRun.Usage 0, // 9: livekit.SimulationRun.mode:type_name -> livekit.SimulationMode - 29, // 10: livekit.Scenario.tags:type_name -> livekit.Scenario.TagsEntry + 30, // 10: livekit.Scenario.tags:type_name -> livekit.Scenario.TagsEntry 5, // 11: livekit.ScenarioGroup.scenarios:type_name -> livekit.Scenario 5, // 12: livekit.SimulationDispatch.scenario:type_name -> livekit.Scenario 0, // 13: livekit.SimulationDispatch.mode:type_name -> livekit.SimulationMode - 33, // 14: livekit.SimulationRunSummary.ChatHistoryEntry.value:type_name -> livekit.agent.ChatContext + 34, // 14: livekit.SimulationRunSummary.ChatHistoryEntry.value:type_name -> livekit.agent.ChatContext 2, // 15: livekit.SimulationRun.Job.status:type_name -> livekit.SimulationRun.Job.Status - 32, // 16: livekit.SimulationRun.Job.started_at:type_name -> google.protobuf.Timestamp - 32, // 17: livekit.SimulationRun.Job.ended_at:type_name -> google.protobuf.Timestamp + 33, // 16: livekit.SimulationRun.Job.started_at:type_name -> google.protobuf.Timestamp + 33, // 17: livekit.SimulationRun.Job.ended_at:type_name -> google.protobuf.Timestamp 17, // 18: livekit.SimulationRun.Job.usage:type_name -> livekit.SimulationRun.Job.Usage - 6, // 19: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup - 0, // 20: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode - 34, // 21: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest - 4, // 22: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun - 1, // 23: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status - 35, // 24: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination - 4, // 25: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun - 35, // 26: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination - 5, // 27: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario - 18, // 28: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request - 20, // 29: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request - 22, // 30: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request - 24, // 31: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request - 26, // 32: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request - 30, // 33: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request - 19, // 34: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response - 21, // 35: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response - 23, // 36: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response - 25, // 37: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response - 27, // 38: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response - 31, // 39: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response - 34, // [34:40] is the sub-list for method output_type - 28, // [28:34] is the sub-list for method input_type - 28, // [28:28] is the sub-list for extension type_name - 28, // [28:28] is the sub-list for extension extendee - 0, // [0:28] is the sub-list for field type_name + 18, // 19: livekit.SimulationRun.Job.audio_metrics:type_name -> livekit.SimulationRun.Job.AudioMetrics + 6, // 20: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup + 0, // 21: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode + 35, // 22: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest + 4, // 23: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun + 1, // 24: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status + 36, // 25: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination + 4, // 26: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun + 36, // 27: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination + 5, // 28: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario + 19, // 29: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request + 21, // 30: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request + 23, // 31: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request + 25, // 32: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request + 27, // 33: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request + 31, // 34: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request + 20, // 35: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response + 22, // 36: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response + 24, // 37: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response + 26, // 38: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response + 28, // 39: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response + 32, // 40: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response + 35, // [35:41] is the sub-list for method output_type + 29, // [29:35] is the sub-list for method input_type + 29, // [29:29] is the sub-list for extension type_name + 29, // [29:29] is the sub-list for extension extendee + 0, // [0:29] is the sub-list for field type_name } func init() { file_livekit_agent_simulation_proto_init() } @@ -2109,15 +2239,15 @@ func file_livekit_agent_simulation_proto_init() { } file_livekit_cloud_agent_proto_init() file_livekit_models_proto_init() - file_livekit_agent_simulation_proto_msgTypes[15].OneofWrappers = []any{} - file_livekit_agent_simulation_proto_msgTypes[21].OneofWrappers = []any{} + file_livekit_agent_simulation_proto_msgTypes[16].OneofWrappers = []any{} + file_livekit_agent_simulation_proto_msgTypes[22].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_livekit_agent_simulation_proto_rawDesc), len(file_livekit_agent_simulation_proto_rawDesc)), NumEnums: 3, - NumMessages: 29, + NumMessages: 30, NumExtensions: 0, NumServices: 1, }, diff --git a/protobufs/livekit_agent_simulation.proto b/protobufs/livekit_agent_simulation.proto index 1322e71c5..ad4303dec 100644 --- a/protobufs/livekit_agent_simulation.proto +++ b/protobufs/livekit_agent_simulation.proto @@ -72,11 +72,31 @@ message SimulationRun { google.protobuf.Timestamp ended_at = 11; string room_id = 12; Usage usage = 13; + // Audio-mode pipeline metrics aggregated over the call's measured turns. + // Only populated for audio jobs. A -1 field means the datum was + // unavailable (Tier-1 metrics require the agent's backchannel). + AudioMetrics audio_metrics = 14; message Usage { int32 text_turns_count = 1; int32 audio_turns_count = 2; } + + message AudioMetrics { + // Tier 0 — true end-to-end response latency off the waveform (our + // playout end -> the agent's speech onset). Always present for audio. + float e2e_latency_p50_s = 1; + float e2e_latency_p95_s = 2; + // Tier 1 — mean over turns; -1 when the backchannel is absent. + float wer = 3; // normalized word error rate of the agent's STT of us + float llm_ttft_s = 4; + float stt_delay_s = 5; + float tts_ttfb_s = 6; + // Call-level barge-in count. + int32 interruptions = 7; + // Turns that contributed measurements (context for the aggregates). + int32 sample_count = 8; + } } message Create { From b8d41c518787cbea606dc6e12f58db3d94447245 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 22:19:12 +0000 Subject: [PATCH 2/5] generated protobuf --- livekit/livekit_agent_simulation.pb.go | 2 +- livekit/livekit_agent_simulation.twirp.go | 241 +++++++++++----------- 2 files changed, 127 insertions(+), 116 deletions(-) diff --git a/livekit/livekit_agent_simulation.pb.go b/livekit/livekit_agent_simulation.pb.go index 4e71d9646..54705f3bd 100644 --- a/livekit/livekit_agent_simulation.pb.go +++ b/livekit/livekit_agent_simulation.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v7.35.0 +// protoc v7.35.1 // source: livekit_agent_simulation.proto package livekit diff --git a/livekit/livekit_agent_simulation.twirp.go b/livekit/livekit_agent_simulation.twirp.go index 3943c0e03..0b1770863 100644 --- a/livekit/livekit_agent_simulation.twirp.go +++ b/livekit/livekit_agent_simulation.twirp.go @@ -1912,119 +1912,130 @@ func (s *agentSimulationServer) PathPrefix() string { } var twirpFileDescriptor9 = []byte{ - // 1818 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x73, 0x1b, 0x49, - 0x15, 0xcf, 0x8c, 0x3e, 0x2c, 0x3d, 0x47, 0xb2, 0xdc, 0xeb, 0x38, 0x93, 0x09, 0xd9, 0x75, 0x1c, - 0x96, 0x75, 0x6d, 0x58, 0x99, 0xf2, 0xb2, 0x2c, 0x09, 0x50, 0x85, 0x2c, 0x2b, 0xb6, 0x82, 0x2d, - 0x9b, 0x91, 0x54, 0x40, 0x28, 0x6a, 0x6a, 0x34, 0xd3, 0x91, 0xc7, 0x91, 0xa6, 0xc5, 0x74, 0x8f, - 0x89, 0x0f, 0x54, 0x71, 0x4c, 0x15, 0x27, 0x8e, 0x5c, 0x28, 0x2e, 0x7b, 0x80, 0x2b, 0x57, 0xaa, - 0xf8, 0x1b, 0xb8, 0xf3, 0x17, 0x70, 0xe7, 0x4e, 0xf5, 0xc7, 0x8c, 0x66, 0xf4, 0x61, 0x7b, 0xb7, - 0x2a, 0x37, 0xf5, 0xeb, 0x5f, 0xbf, 0x7e, 0xfd, 0xfa, 0xf7, 0x7e, 0xaf, 0x47, 0xf0, 0xe1, 0xc8, - 0xbf, 0xc4, 0x6f, 0x7c, 0x66, 0x3b, 0x43, 0x1c, 0x30, 0x9b, 0xfa, 0xe3, 0x68, 0xe4, 0x30, 0x9f, - 0x04, 0xf5, 0x49, 0x48, 0x18, 0x41, 0x2b, 0x6a, 0xde, 0x7c, 0x2c, 0x00, 0xbb, 0x33, 0x70, 0x4c, - 0x69, 0x82, 0x35, 0x3f, 0x1a, 0x12, 0x32, 0x1c, 0xe1, 0x5d, 0x31, 0x1a, 0x44, 0xaf, 0x77, 0x99, - 0x3f, 0xc6, 0x94, 0x39, 0xe3, 0x89, 0x02, 0x3c, 0x88, 0x57, 0xbb, 0x23, 0x12, 0x79, 0xd2, 0x87, - 0x9a, 0xda, 0x88, 0xa7, 0xc6, 0xc4, 0xc3, 0x23, 0x2a, 0xad, 0xdb, 0xff, 0xce, 0xc1, 0x46, 0x37, - 0x09, 0xc9, 0x8a, 0x82, 0x6e, 0x34, 0x1e, 0x3b, 0xe1, 0x15, 0xda, 0x84, 0xe2, 0xc4, 0xa1, 0x14, - 0x7b, 0x86, 0xb6, 0xa5, 0xed, 0x14, 0x2c, 0x35, 0xe2, 0xf6, 0xd7, 0x8e, 0x3f, 0xc2, 0x9e, 0xa1, - 0x4b, 0xbb, 0x1c, 0xa1, 0x47, 0x00, 0x43, 0xe2, 0x07, 0x43, 0xfb, 0x77, 0x78, 0x34, 0x32, 0x72, - 0x5b, 0xda, 0x4e, 0xd9, 0x2a, 0x0b, 0xcb, 0x2f, 0xf0, 0x68, 0xc4, 0xa7, 0x19, 0xb1, 0xfd, 0xf1, - 0x24, 0x24, 0x97, 0xd8, 0xc8, 0xcb, 0x69, 0x46, 0xda, 0xd2, 0x80, 0x7e, 0x04, 0x45, 0x9f, 0xd2, - 0x08, 0x53, 0xa3, 0xb0, 0x95, 0xdb, 0x59, 0xdd, 0x7b, 0x52, 0x57, 0xd1, 0xd6, 0x17, 0x05, 0x57, - 0x6f, 0x73, 0xac, 0xa5, 0x96, 0xa0, 0x9f, 0xc3, 0x5d, 0xf7, 0xdc, 0x61, 0xf6, 0xb9, 0x4f, 0x19, - 0x09, 0xaf, 0x8c, 0xa2, 0x70, 0x51, 0xbf, 0xde, 0x45, 0xf3, 0xdc, 0x61, 0x47, 0x72, 0x41, 0x2b, - 0x60, 0xe1, 0x95, 0xb5, 0xea, 0x4e, 0x2d, 0xe6, 0x2b, 0xa8, 0xcd, 0x02, 0x50, 0x0d, 0x72, 0x6f, - 0xf0, 0x95, 0x48, 0x47, 0xd9, 0xe2, 0x3f, 0xd1, 0xf7, 0xa0, 0x70, 0xe9, 0x8c, 0x22, 0x2c, 0x52, - 0xb1, 0xba, 0x67, 0x26, 0x3b, 0xca, 0xbc, 0x73, 0x0f, 0x4d, 0x12, 0x30, 0xfc, 0x96, 0x59, 0x12, - 0xf8, 0x5c, 0xff, 0xa1, 0x66, 0xda, 0x50, 0x10, 0xf1, 0xa3, 0x2d, 0x58, 0xf5, 0x30, 0x75, 0x43, - 0x7f, 0xc2, 0x63, 0x53, 0x8e, 0xd3, 0x26, 0xf4, 0x21, 0x00, 0x8d, 0x86, 0x43, 0x4c, 0x05, 0x40, - 0x17, 0x80, 0x94, 0x05, 0x6d, 0x40, 0x61, 0xe4, 0x0c, 0x70, 0x9c, 0x6f, 0x39, 0xd8, 0xfe, 0xc3, - 0x7d, 0xa8, 0x64, 0xce, 0x8c, 0xaa, 0xa0, 0xfb, 0x9e, 0xda, 0x40, 0xf7, 0xc5, 0x65, 0x4d, 0x42, - 0x72, 0x81, 0x5d, 0x66, 0xfb, 0x9e, 0xf2, 0x5b, 0x56, 0x96, 0xb6, 0x87, 0xbe, 0x80, 0x22, 0x65, - 0x0e, 0x8b, 0xa8, 0xf0, 0x5b, 0xdd, 0x7b, 0xb4, 0x38, 0x95, 0xf5, 0xae, 0x00, 0x59, 0x0a, 0x8c, - 0x9e, 0xc2, 0xba, 0x24, 0x6d, 0xfa, 0x54, 0xf2, 0xaa, 0x6b, 0x62, 0xe2, 0x20, 0x75, 0xb4, 0x0d, - 0x28, 0xe0, 0x30, 0x24, 0xa1, 0x51, 0x90, 0xa1, 0x8b, 0x01, 0x7a, 0x06, 0xe0, 0x86, 0xd8, 0x61, - 0xd8, 0xb3, 0x1d, 0x66, 0x14, 0x55, 0x5a, 0x25, 0xeb, 0xeb, 0x31, 0xeb, 0xeb, 0xbd, 0x98, 0xf5, - 0x56, 0x59, 0xa1, 0x1b, 0x0c, 0xd5, 0x21, 0x7f, 0x41, 0x06, 0xd4, 0x58, 0x11, 0xb7, 0x6f, 0x2e, - 0x09, 0xf9, 0x25, 0x19, 0x58, 0x02, 0x87, 0xbe, 0x84, 0x15, 0x2a, 0xb9, 0x60, 0x94, 0xc4, 0x3e, - 0x8f, 0xae, 0x25, 0x8c, 0x15, 0xa3, 0x79, 0xf2, 0xe4, 0x31, 0x03, 0x67, 0x8c, 0x8d, 0xb2, 0x4c, - 0x9e, 0xb0, 0x74, 0x9c, 0x31, 0x46, 0x3f, 0x81, 0x2a, 0x75, 0x71, 0xe0, 0x84, 0x3e, 0xb1, 0x87, - 0x21, 0x89, 0x26, 0x06, 0x08, 0xf7, 0x9b, 0x53, 0xf7, 0x6a, 0xfa, 0x90, 0xcf, 0x5a, 0x15, 0x9a, - 0x1e, 0xa2, 0x2f, 0xa0, 0x84, 0x03, 0x4f, 0x9e, 0x7f, 0xf5, 0xc6, 0xf3, 0xaf, 0x08, 0x6c, 0x83, - 0xa1, 0x87, 0x50, 0xbe, 0x20, 0x03, 0xdb, 0x25, 0x51, 0xc0, 0x8c, 0xbb, 0xa2, 0x32, 0x4b, 0x17, - 0x64, 0xd0, 0xe4, 0x63, 0xf4, 0x18, 0xee, 0xca, 0xea, 0x55, 0xf3, 0x15, 0x31, 0xbf, 0x2a, 0x6d, - 0x09, 0x44, 0x16, 0xb2, 0x82, 0x54, 0x25, 0x44, 0xda, 0x24, 0xe4, 0x13, 0x58, 0x0b, 0xa2, 0x71, - 0x4a, 0xc0, 0xa8, 0xb1, 0x26, 0x50, 0xd5, 0x20, 0x1a, 0x4f, 0x53, 0x46, 0xd1, 0x1e, 0x14, 0x22, - 0xea, 0x0c, 0xb1, 0x51, 0x13, 0xf1, 0x7f, 0x6b, 0xc9, 0x55, 0xf4, 0x39, 0xc6, 0x92, 0x50, 0x5e, - 0x0b, 0x2e, 0x09, 0xdc, 0x28, 0x0c, 0x71, 0xe0, 0x5e, 0x19, 0xeb, 0x72, 0xfb, 0x94, 0x09, 0x3d, - 0x85, 0x3c, 0x57, 0x2e, 0x03, 0x09, 0x4a, 0xde, 0x5f, 0xe0, 0xf4, 0x84, 0x78, 0xd8, 0x12, 0x20, - 0xf3, 0xcf, 0x05, 0xc8, 0xbd, 0x24, 0x83, 0x39, 0xe2, 0x3f, 0x4b, 0x98, 0xad, 0x0b, 0x37, 0x8f, - 0x97, 0xd3, 0x64, 0x96, 0xdd, 0xdb, 0x70, 0xd7, 0x0f, 0x28, 0x0b, 0x23, 0x57, 0x9e, 0x5d, 0x96, - 0x5c, 0xc6, 0x36, 0x25, 0x75, 0x3e, 0x4d, 0xea, 0xcf, 0x00, 0x49, 0xc2, 0xe0, 0xb7, 0x13, 0xec, - 0x32, 0x95, 0x3b, 0xc9, 0x7b, 0x59, 0x31, 0xad, 0xd4, 0xc4, 0xb4, 0xa8, 0x8b, 0xa9, 0xa2, 0x46, - 0x08, 0xf2, 0xcc, 0x19, 0x4a, 0x7a, 0x97, 0x2d, 0xf1, 0x9b, 0x5f, 0x7a, 0x48, 0xc8, 0x58, 0x12, - 0xb1, 0x24, 0xd0, 0x25, 0x6e, 0x10, 0x3c, 0x7c, 0x06, 0x40, 0x99, 0x13, 0xaa, 0x52, 0x82, 0x9b, - 0x4b, 0x49, 0xa1, 0x1b, 0xec, 0x9b, 0x72, 0xf0, 0x3e, 0xac, 0x88, 0x70, 0x7c, 0x4f, 0x30, 0xb0, - 0x6c, 0x15, 0xf9, 0xb0, 0xed, 0xa1, 0x1f, 0xc4, 0x84, 0xa8, 0x08, 0x67, 0x5b, 0xd7, 0x24, 0x3d, - 0x4d, 0x0a, 0xf3, 0x37, 0x50, 0x10, 0x63, 0xb4, 0x03, 0x35, 0xae, 0xa2, 0x36, 0x8b, 0xc2, 0x80, - 0x2a, 0x86, 0xca, 0xb6, 0x54, 0xe5, 0xf6, 0x1e, 0x37, 0x4b, 0x92, 0x7e, 0x0a, 0xeb, 0x4e, 0xe4, - 0xf9, 0x24, 0x03, 0x95, 0x9d, 0x6a, 0x4d, 0x4c, 0x4c, 0xb1, 0xdb, 0x04, 0x8a, 0xf2, 0x8e, 0x11, - 0x82, 0x6a, 0xb7, 0xd7, 0xe8, 0xf5, 0xbb, 0xf6, 0x59, 0xab, 0x73, 0xd0, 0xee, 0x1c, 0xd6, 0xee, - 0xa4, 0x6c, 0x56, 0xbf, 0xd3, 0xe1, 0x36, 0x0d, 0x6d, 0x40, 0x4d, 0xd9, 0x9a, 0xa7, 0x27, 0x67, - 0xc7, 0xad, 0x5e, 0xeb, 0xa0, 0xa6, 0xa3, 0x75, 0xa8, 0x28, 0xeb, 0x8b, 0x46, 0xfb, 0xb8, 0x75, - 0x50, 0xcb, 0xa5, 0x81, 0x8d, 0x4e, 0xb3, 0x75, 0xcc, 0xad, 0xf9, 0x97, 0xf9, 0x52, 0xb9, 0x06, - 0xe6, 0xff, 0x72, 0x50, 0x6c, 0x0a, 0xd9, 0x32, 0xff, 0xa3, 0xc3, 0x8a, 0x85, 0x7f, 0x1b, 0x61, - 0xca, 0x66, 0x34, 0x59, 0x9b, 0xd5, 0xe4, 0xac, 0xea, 0xe8, 0xb3, 0xaa, 0xb3, 0xa0, 0x38, 0xf3, - 0x0b, 0x8b, 0x73, 0x13, 0x8a, 0x21, 0x1e, 0x72, 0x65, 0x2e, 0xaa, 0x3b, 0x12, 0x23, 0xb4, 0x3f, - 0x27, 0x5b, 0x2b, 0xd7, 0xc9, 0xd6, 0xd1, 0x9d, 0x19, 0xe1, 0x7a, 0xa7, 0x69, 0xe8, 0xe3, 0x6c, - 0x11, 0x73, 0x46, 0x16, 0x8e, 0xb4, 0x4c, 0x19, 0x73, 0x58, 0x5c, 0xc9, 0xe5, 0x5b, 0x54, 0xf2, - 0xfe, 0x3a, 0xac, 0xd9, 0xd9, 0xc0, 0xf6, 0xab, 0x70, 0xd7, 0x4e, 0xb9, 0x7c, 0x99, 0x2f, 0xe5, - 0x6a, 0x79, 0x6b, 0xbe, 0xf7, 0x98, 0x7f, 0xd4, 0xa0, 0x64, 0x61, 0x3a, 0x21, 0x01, 0xc5, 0x9c, - 0x19, 0xd3, 0xec, 0xd8, 0x61, 0x14, 0x4c, 0xd3, 0xbc, 0x46, 0xd3, 0x3c, 0x6c, 0x7b, 0xa8, 0x0b, - 0x9b, 0x93, 0x10, 0x53, 0x7f, 0x18, 0x60, 0xcf, 0x9e, 0x10, 0xca, 0xec, 0x50, 0xde, 0x92, 0xea, - 0xf4, 0xd3, 0x56, 0x71, 0x16, 0xc3, 0xce, 0x08, 0x65, 0xea, 0x2a, 0xad, 0x8d, 0xc9, 0x02, 0xab, - 0xf9, 0x57, 0x0d, 0x3e, 0x68, 0x92, 0xe0, 0xb5, 0x1f, 0x8e, 0xbb, 0x24, 0x0a, 0x5d, 0xdc, 0x9f, - 0x8c, 0x88, 0xe3, 0x99, 0xbf, 0xbf, 0x35, 0x07, 0x16, 0x1e, 0x41, 0x5f, 0x7c, 0x84, 0x4f, 0x60, - 0xcd, 0x25, 0x1e, 0xb6, 0x31, 0x7f, 0xbb, 0x4c, 0x88, 0x1f, 0x30, 0xa5, 0x58, 0x55, 0x6e, 0x6e, - 0x25, 0x56, 0x13, 0xa6, 0x39, 0x32, 0xff, 0xa4, 0x41, 0xee, 0x10, 0x33, 0xb3, 0xf7, 0x3e, 0x42, - 0x32, 0xbf, 0x9f, 0xba, 0x8d, 0x1d, 0xc8, 0x85, 0x91, 0x7c, 0xf3, 0x64, 0x38, 0x96, 0x5e, 0x62, - 0x71, 0x88, 0xf9, 0x2f, 0x1d, 0xf2, 0xc7, 0x3e, 0x65, 0xe6, 0x3f, 0xb5, 0x5b, 0x47, 0xf5, 0x7c, - 0x46, 0xe6, 0xaf, 0x7f, 0xc0, 0x1c, 0xdd, 0x89, 0x45, 0x9e, 0xb3, 0xf3, 0xc7, 0x00, 0x13, 0x67, - 0x88, 0x6d, 0x46, 0xde, 0xe0, 0x40, 0xe4, 0x6c, 0x75, 0xcf, 0x48, 0xd6, 0xf7, 0xb8, 0xf5, 0xcc, - 0x19, 0xfa, 0x81, 0x70, 0x72, 0xa4, 0x59, 0x65, 0x8e, 0x16, 0xe6, 0x77, 0x9a, 0xb6, 0x5f, 0x86, - 0x15, 0x5b, 0xfa, 0xda, 0xaf, 0xc0, 0xaa, 0x3d, 0xf5, 0x64, 0xbe, 0xcd, 0x70, 0x31, 0x1f, 0x46, - 0x01, 0x35, 0x34, 0xf1, 0x56, 0x59, 0x76, 0x7c, 0x81, 0x41, 0x3f, 0x85, 0xb5, 0x80, 0x6b, 0x5f, - 0x2a, 0x28, 0xfd, 0xfa, 0xa0, 0xac, 0x0a, 0x5f, 0x70, 0x16, 0x87, 0x65, 0x5e, 0x40, 0xb1, 0xe9, - 0x04, 0x2e, 0x1e, 0xbd, 0xa7, 0x7b, 0x4d, 0x33, 0xe8, 0x3d, 0x4b, 0xf6, 0x57, 0x5a, 0xa2, 0xd9, - 0x0f, 0xe0, 0x5e, 0x56, 0xb3, 0xed, 0xfe, 0xd9, 0xf1, 0x69, 0xe3, 0xa0, 0x76, 0x07, 0xdd, 0x83, - 0x75, 0x35, 0x75, 0xd8, 0xea, 0xb4, 0xac, 0x46, 0x4f, 0xaa, 0xf7, 0xbc, 0xa2, 0xeb, 0x68, 0x13, - 0x90, 0xb2, 0x75, 0xfb, 0x27, 0x27, 0x0d, 0xab, 0xfd, 0x8a, 0xdb, 0x73, 0x0b, 0x95, 0x3e, 0x3f, - 0xaf, 0xf4, 0x85, 0x85, 0x4a, 0x5f, 0xdc, 0xfe, 0x2a, 0x07, 0xa5, 0x58, 0x2f, 0xa7, 0x0d, 0x5d, - 0x4b, 0x37, 0xf4, 0xd9, 0xf7, 0x84, 0xbe, 0xe0, 0x3d, 0xb1, 0xf8, 0xe5, 0x90, 0x5b, 0xf6, 0x72, - 0xd8, 0x55, 0x6f, 0x84, 0xbc, 0xa0, 0xd5, 0xc3, 0x39, 0xe5, 0xae, 0xf7, 0x9c, 0x21, 0x95, 0x5f, - 0x3b, 0xf2, 0x01, 0x61, 0x42, 0x29, 0xa2, 0x38, 0xf4, 0x1c, 0xe6, 0xa8, 0xf7, 0x48, 0x32, 0x36, - 0xff, 0xa6, 0xc1, 0xba, 0x6c, 0x53, 0x2f, 0x42, 0x32, 0xee, 0xca, 0xef, 0x50, 0xf3, 0x57, 0xb7, - 0x66, 0x50, 0xea, 0x35, 0xa0, 0x67, 0x5e, 0x03, 0xd3, 0x0e, 0x94, 0x4b, 0x77, 0x20, 0xf3, 0x59, - 0xaa, 0x40, 0x3e, 0x83, 0x52, 0x2c, 0xfa, 0x4a, 0x23, 0xd6, 0xe7, 0x4e, 0x63, 0x25, 0x10, 0xf3, - 0x4b, 0x28, 0x27, 0x47, 0x5b, 0xf0, 0x9d, 0xb6, 0x91, 0xfe, 0x4e, 0x2b, 0xa7, 0xbe, 0xc5, 0xb6, - 0x7b, 0x50, 0xc9, 0xb4, 0x35, 0xfe, 0xcc, 0x12, 0x0d, 0x56, 0xae, 0x16, 0xbf, 0xd1, 0x2e, 0x94, - 0xe3, 0x9d, 0xf8, 0x35, 0xe5, 0x16, 0x47, 0x33, 0xc5, 0x6c, 0xff, 0x43, 0x03, 0x34, 0x2d, 0xe5, - 0x03, 0x9f, 0x4e, 0x1c, 0xe6, 0x9e, 0x7f, 0xad, 0x0e, 0x74, 0x0f, 0x8a, 0xfc, 0x3d, 0x9f, 0x24, - 0xaf, 0x70, 0x41, 0x06, 0x6d, 0x2f, 0x93, 0x97, 0xdc, 0x8d, 0x79, 0x49, 0x3a, 0x6d, 0xfe, 0x16, - 0x9d, 0xf6, 0xd3, 0xd7, 0x50, 0xcd, 0xda, 0xd1, 0x47, 0xf0, 0xb0, 0xdb, 0x3e, 0xe9, 0x1f, 0x37, - 0x7a, 0xed, 0xd3, 0x8e, 0x7d, 0x72, 0x7a, 0xd0, 0xb2, 0xfb, 0x9d, 0xee, 0x59, 0xab, 0xd9, 0x7e, - 0xd1, 0x6e, 0xf1, 0x42, 0x33, 0x60, 0x63, 0x16, 0xd0, 0x6b, 0xfd, 0xb2, 0x57, 0xd3, 0x44, 0x75, - 0xce, 0xcc, 0x34, 0xfa, 0x07, 0xed, 0xd3, 0x9a, 0xbe, 0xf7, 0x97, 0x02, 0xac, 0x35, 0x38, 0x77, - 0xa7, 0xbb, 0x21, 0x0f, 0x3e, 0x90, 0x5c, 0xcb, 0x7e, 0xb7, 0x7e, 0xbc, 0x44, 0xb7, 0x25, 0xb6, - 0x1e, 0x77, 0xd6, 0xef, 0xdc, 0x04, 0x53, 0xac, 0x7a, 0xa7, 0xc1, 0xa3, 0xb8, 0x03, 0x27, 0xc0, - 0x74, 0x2f, 0x46, 0x7b, 0xcb, 0x3c, 0xcd, 0xf7, 0xed, 0x64, 0xf7, 0xcf, 0xbf, 0xd6, 0x1a, 0x15, - 0xca, 0xaf, 0xa1, 0x76, 0x88, 0x59, 0xf6, 0xb4, 0xdb, 0x4b, 0x1c, 0x1d, 0x62, 0x96, 0x6c, 0xf6, - 0xe4, 0x5a, 0x8c, 0x72, 0x6e, 0x03, 0xe2, 0x1d, 0x33, 0x83, 0xa0, 0x68, 0xd9, 0x52, 0x0e, 0x4d, - 0xfc, 0x7f, 0xfb, 0x7a, 0x90, 0xda, 0x80, 0x5f, 0x97, 0xe8, 0x28, 0xb7, 0xbc, 0x2e, 0x81, 0xbd, - 0xf9, 0xba, 0x62, 0x98, 0xda, 0xe5, 0x12, 0x1e, 0x28, 0x52, 0x28, 0x3e, 0xa7, 0x84, 0x08, 0x3d, - 0x9d, 0x57, 0xb7, 0x39, 0xb5, 0x4a, 0x76, 0xfc, 0xee, 0xed, 0xc0, 0x72, 0xdf, 0xfd, 0x17, 0xaf, - 0x9e, 0x0c, 0x7d, 0x76, 0x1e, 0x0d, 0xea, 0x2e, 0x19, 0xc7, 0xff, 0xc7, 0xc9, 0xff, 0xdc, 0x5c, - 0x32, 0x8a, 0x0d, 0x7f, 0xd7, 0x2b, 0xc7, 0xfe, 0x25, 0xfe, 0x99, 0x78, 0x04, 0x12, 0x46, 0xfe, - 0xab, 0x57, 0xd5, 0xf8, 0xf9, 0x73, 0x61, 0x18, 0x14, 0xc5, 0x92, 0xcf, 0xff, 0x1f, 0x00, 0x00, - 0xff, 0xff, 0x8e, 0x7f, 0x46, 0xe2, 0x02, 0x14, 0x00, 0x00, + // 1993 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x6f, 0xdb, 0xc8, + 0x15, 0x0f, 0xa9, 0x0f, 0x4b, 0x4f, 0x96, 0x2c, 0xcf, 0x3a, 0x59, 0x85, 0xd9, 0x64, 0x1d, 0x67, + 0xb7, 0x71, 0x37, 0x5d, 0x39, 0xd0, 0xd6, 0xdd, 0x3a, 0x6d, 0x81, 0xca, 0xb6, 0x62, 0x2b, 0xf5, + 0x57, 0x29, 0x19, 0x6d, 0x53, 0x14, 0x04, 0x45, 0x8e, 0x65, 0x3a, 0x14, 0x47, 0xe5, 0x0c, 0xbd, + 0xf1, 0xa1, 0xc7, 0x02, 0x01, 0x7a, 0xea, 0xb9, 0x40, 0xd1, 0xcb, 0x1e, 0xda, 0x6b, 0xaf, 0x05, + 0xfa, 0x37, 0xf4, 0xde, 0xbf, 0xa0, 0xe8, 0xb5, 0xf7, 0x62, 0x3e, 0x28, 0x91, 0x92, 0x2c, 0x7b, + 0x17, 0xc8, 0x4d, 0xf3, 0x9b, 0xdf, 0xbc, 0x79, 0xf3, 0xe6, 0xf7, 0xde, 0x1b, 0x0a, 0x1e, 0xf9, + 0xde, 0x25, 0x7e, 0xe3, 0x31, 0xcb, 0xee, 0xe3, 0x80, 0x59, 0xd4, 0x1b, 0x44, 0xbe, 0xcd, 0x3c, + 0x12, 0xd4, 0x87, 0x21, 0x61, 0x04, 0x2d, 0xa8, 0x79, 0xe3, 0xb1, 0x20, 0x6c, 0x4c, 0xd0, 0x31, + 0xa5, 0x23, 0xae, 0xf1, 0x71, 0x9f, 0x90, 0xbe, 0x8f, 0x37, 0xc4, 0xa8, 0x17, 0x9d, 0x6d, 0x30, + 0x6f, 0x80, 0x29, 0xb3, 0x07, 0x43, 0x45, 0xb8, 0x1f, 0xaf, 0x76, 0x7c, 0x12, 0xb9, 0xd2, 0x86, + 0x9a, 0x5a, 0x89, 0xa7, 0x06, 0xc4, 0xc5, 0x3e, 0x95, 0xe8, 0xda, 0xbf, 0x32, 0xb0, 0xd2, 0x19, + 0xb9, 0x64, 0x46, 0x41, 0x27, 0x1a, 0x0c, 0xec, 0xf0, 0x0a, 0xdd, 0x83, 0xfc, 0xd0, 0xa6, 0x14, + 0xbb, 0x35, 0x6d, 0x55, 0x5b, 0xcf, 0x99, 0x6a, 0xc4, 0xf1, 0x33, 0xdb, 0xf3, 0xb1, 0x5b, 0xd3, + 0x25, 0x2e, 0x47, 0xe8, 0x21, 0x40, 0x9f, 0x78, 0x41, 0xdf, 0xfa, 0x0a, 0xfb, 0x7e, 0x2d, 0xb3, + 0xaa, 0xad, 0x17, 0xcd, 0xa2, 0x40, 0x7e, 0x81, 0x7d, 0x9f, 0x4f, 0x33, 0x62, 0x79, 0x83, 0x61, + 0x48, 0x2e, 0x71, 0x2d, 0x2b, 0xa7, 0x19, 0x69, 0x4b, 0x00, 0xfd, 0x08, 0xf2, 0x1e, 0xa5, 0x11, + 0xa6, 0xb5, 0xdc, 0x6a, 0x66, 0xbd, 0xd4, 0x78, 0x52, 0x57, 0xde, 0xd6, 0x67, 0x39, 0x57, 0x6f, + 0x73, 0xae, 0xa9, 0x96, 0xa0, 0x9f, 0xc3, 0xa2, 0x73, 0x6e, 0x33, 0xeb, 0xdc, 0xa3, 0x8c, 0x84, + 0x57, 0xb5, 0xbc, 0x30, 0x51, 0x9f, 0x6f, 0x62, 0xe7, 0xdc, 0x66, 0xfb, 0x72, 0x41, 0x2b, 0x60, + 0xe1, 0x95, 0x59, 0x72, 0xc6, 0x88, 0xf1, 0x1a, 0xaa, 0x93, 0x04, 0x54, 0x85, 0xcc, 0x1b, 0x7c, + 0x25, 0xc2, 0x51, 0x34, 0xf9, 0x4f, 0xf4, 0x1c, 0x72, 0x97, 0xb6, 0x1f, 0x61, 0x11, 0x8a, 0x52, + 0xc3, 0x18, 0xed, 0x28, 0xe3, 0xce, 0x2d, 0xec, 0x90, 0x80, 0xe1, 0xb7, 0xcc, 0x94, 0xc4, 0x17, + 0xfa, 0x0f, 0x35, 0xc3, 0x82, 0x9c, 0xf0, 0x1f, 0xad, 0x42, 0xc9, 0xc5, 0xd4, 0x09, 0xbd, 0x21, + 0xf7, 0x4d, 0x19, 0x4e, 0x42, 0xe8, 0x11, 0x00, 0x8d, 0xfa, 0x7d, 0x4c, 0x05, 0x41, 0x17, 0x84, + 0x04, 0x82, 0x56, 0x20, 0xe7, 0xdb, 0x3d, 0x1c, 0xc7, 0x5b, 0x0e, 0xd6, 0xfe, 0x7b, 0x1f, 0xca, + 0xa9, 0x33, 0xa3, 0x0a, 0xe8, 0x9e, 0xab, 0x36, 0xd0, 0x3d, 0x71, 0x59, 0xc3, 0x90, 0x5c, 0x60, + 0x87, 0x59, 0x9e, 0xab, 0xec, 0x16, 0x15, 0xd2, 0x76, 0xd1, 0x26, 0xe4, 0x29, 0xb3, 0x59, 0x44, + 0x85, 0xdd, 0x4a, 0xe3, 0xe1, 0xec, 0x50, 0xd6, 0x3b, 0x82, 0x64, 0x2a, 0x32, 0x7a, 0x06, 0xcb, + 0x52, 0xb4, 0xc9, 0x53, 0xc9, 0xab, 0xae, 0x8a, 0x89, 0xdd, 0xc4, 0xd1, 0x56, 0x20, 0x87, 0xc3, + 0x90, 0x84, 0xb5, 0x9c, 0x74, 0x5d, 0x0c, 0xd0, 0x16, 0x80, 0x13, 0x62, 0x9b, 0x61, 0xd7, 0xb2, + 0x59, 0x2d, 0xaf, 0xc2, 0x2a, 0x55, 0x5f, 0x8f, 0x55, 0x5f, 0xef, 0xc6, 0xaa, 0x37, 0x8b, 0x8a, + 0xdd, 0x64, 0xa8, 0x0e, 0xd9, 0x0b, 0xd2, 0xa3, 0xb5, 0x05, 0x71, 0xfb, 0xc6, 0x35, 0x2e, 0xbf, + 0x22, 0x3d, 0x53, 0xf0, 0xd0, 0x97, 0xb0, 0x40, 0xa5, 0x16, 0x6a, 0x05, 0xb1, 0xcf, 0xc3, 0xb9, + 0x82, 0x31, 0x63, 0x36, 0x0f, 0x9e, 0x3c, 0x66, 0x60, 0x0f, 0x70, 0xad, 0x28, 0x83, 0x27, 0x90, + 0x23, 0x7b, 0x80, 0xd1, 0x4f, 0xa0, 0x42, 0x1d, 0x1c, 0xd8, 0xa1, 0x47, 0xac, 0x7e, 0x48, 0xa2, + 0x61, 0x0d, 0x84, 0xf9, 0x7b, 0x63, 0xf3, 0x6a, 0x7a, 0x8f, 0xcf, 0x9a, 0x65, 0x9a, 0x1c, 0xa2, + 0x4d, 0x28, 0xe0, 0xc0, 0x95, 0xe7, 0x2f, 0xdd, 0x78, 0xfe, 0x05, 0xc1, 0x6d, 0x32, 0xf4, 0x00, + 0x8a, 0x17, 0xa4, 0x67, 0x39, 0x24, 0x0a, 0x58, 0x6d, 0x51, 0x64, 0x66, 0xe1, 0x82, 0xf4, 0x76, + 0xf8, 0x18, 0x3d, 0x86, 0x45, 0x99, 0xbd, 0x6a, 0xbe, 0x2c, 0xe6, 0x4b, 0x12, 0x1b, 0x51, 0x64, + 0x22, 0x2b, 0x4a, 0x45, 0x52, 0x24, 0x26, 0x29, 0x4f, 0x61, 0x29, 0x88, 0x06, 0x89, 0x02, 0x46, + 0x6b, 0x4b, 0x82, 0x55, 0x09, 0xa2, 0xc1, 0x38, 0x64, 0x14, 0x35, 0x20, 0x17, 0x51, 0xbb, 0x8f, + 0x6b, 0x55, 0xe1, 0xff, 0x47, 0xd7, 0x5c, 0xc5, 0x29, 0xe7, 0x98, 0x92, 0xca, 0x73, 0xc1, 0x21, + 0x81, 0x13, 0x85, 0x21, 0x0e, 0x9c, 0xab, 0xda, 0xb2, 0xdc, 0x3e, 0x01, 0xa1, 0x67, 0x90, 0xe5, + 0x95, 0xab, 0x86, 0x84, 0x24, 0x3f, 0x9c, 0x61, 0xf4, 0x90, 0xb8, 0xd8, 0x14, 0x24, 0xe3, 0xf7, + 0x05, 0xc8, 0xbc, 0x22, 0xbd, 0x29, 0xe1, 0x6f, 0x8d, 0x94, 0xad, 0x0b, 0x33, 0x8f, 0xaf, 0x97, + 0xc9, 0xa4, 0xba, 0xd7, 0x60, 0xd1, 0x0b, 0x28, 0x0b, 0x23, 0x47, 0x9e, 0x5d, 0xa6, 0x5c, 0x0a, + 0x1b, 0x8b, 0x3a, 0x9b, 0x14, 0xf5, 0xe7, 0x80, 0xa4, 0x60, 0xf0, 0xdb, 0x21, 0x76, 0x98, 0x8a, + 0x9d, 0xd4, 0xbd, 0xcc, 0x98, 0x56, 0x62, 0x62, 0x9c, 0xd4, 0xf9, 0x44, 0x52, 0x23, 0x04, 0x59, + 0x66, 0xf7, 0xa5, 0xbc, 0x8b, 0xa6, 0xf8, 0xcd, 0x2f, 0x3d, 0x24, 0x64, 0x20, 0x85, 0x58, 0x10, + 0xec, 0x02, 0x07, 0x84, 0x0e, 0xb7, 0x00, 0x28, 0xb3, 0x43, 0x95, 0x4a, 0x70, 0x73, 0x2a, 0x29, + 0x76, 0x93, 0x7d, 0x5b, 0x0d, 0x7e, 0x08, 0x0b, 0xc2, 0x1d, 0xcf, 0x15, 0x0a, 0x2c, 0x9a, 0x79, + 0x3e, 0x6c, 0xbb, 0xe8, 0x07, 0xb1, 0x20, 0xca, 0xc2, 0xd8, 0xea, 0x9c, 0xa0, 0xa7, 0x44, 0x71, + 0x00, 0x65, 0x3b, 0x72, 0x3d, 0x62, 0x0d, 0x30, 0x0b, 0x3d, 0x87, 0x0a, 0x55, 0x96, 0x1a, 0x4f, + 0xe7, 0xac, 0x6f, 0x72, 0xfe, 0xa1, 0xa4, 0x9b, 0x8b, 0x76, 0x62, 0x64, 0xfc, 0x06, 0x72, 0xc2, + 0x3a, 0x5a, 0x87, 0x2a, 0xaf, 0xc9, 0x16, 0x8b, 0xc2, 0x80, 0x2a, 0xbd, 0xcb, 0x26, 0x57, 0xe1, + 0x78, 0x97, 0xc3, 0x52, 0xf2, 0x9f, 0xc1, 0xb2, 0x74, 0x20, 0x49, 0x95, 0x7d, 0x6f, 0x49, 0x4c, + 0x8c, 0xb9, 0xc6, 0x9f, 0x74, 0x58, 0x4c, 0xee, 0x8e, 0xbe, 0x0b, 0xcb, 0xb8, 0x81, 0x2d, 0xdf, + 0x66, 0x5c, 0xbf, 0xd6, 0x70, 0xf3, 0xb9, 0x45, 0xc5, 0x3e, 0xba, 0x59, 0xc1, 0x0d, 0x7c, 0x20, + 0xf1, 0x93, 0xcd, 0xe7, 0x9d, 0x29, 0xea, 0xd6, 0xa6, 0x25, 0x15, 0x9a, 0xa6, 0x6e, 0x6d, 0x76, + 0x78, 0x17, 0xfa, 0x0a, 0x87, 0x42, 0x7d, 0xba, 0xc9, 0x7f, 0xa2, 0x8f, 0x00, 0x7c, 0x7f, 0x60, + 0x31, 0x76, 0xc6, 0x2c, 0x2a, 0x94, 0xa7, 0x9b, 0x05, 0xdf, 0x1f, 0x74, 0xd9, 0x19, 0xeb, 0xa0, + 0x47, 0x50, 0xa2, 0x8c, 0x97, 0x64, 0xdf, 0xbe, 0xb2, 0xa4, 0xea, 0x74, 0x7e, 0xd7, 0x6c, 0x97, + 0x23, 0x1d, 0xbe, 0x9a, 0x31, 0xca, 0x57, 0xf7, 0x2c, 0x2a, 0x24, 0xa7, 0x9b, 0x05, 0xc6, 0x68, + 0x97, 0x9d, 0xf5, 0x3a, 0xe8, 0x13, 0x28, 0x7b, 0x01, 0xc3, 0x61, 0x18, 0x0d, 0xa5, 0x6a, 0x17, + 0xc4, 0xe1, 0xd3, 0x20, 0x2f, 0x1e, 0xd4, 0x1e, 0x0c, 0x7d, 0xac, 0x22, 0x54, 0x90, 0xd9, 0x2b, + 0x31, 0x11, 0x9d, 0x35, 0x02, 0x79, 0x99, 0x4f, 0x08, 0x41, 0xa5, 0xd3, 0x6d, 0x76, 0x4f, 0x3b, + 0xd6, 0x49, 0xeb, 0x68, 0xb7, 0x7d, 0xb4, 0x57, 0xbd, 0x93, 0xc0, 0xcc, 0xd3, 0xa3, 0x23, 0x8e, + 0x69, 0x68, 0x05, 0xaa, 0x0a, 0xdb, 0x39, 0x3e, 0x3c, 0x39, 0x68, 0x75, 0x5b, 0xbb, 0x55, 0x1d, + 0x2d, 0x43, 0x59, 0xa1, 0x2f, 0x9b, 0xed, 0x83, 0xd6, 0x6e, 0x35, 0x93, 0x24, 0x36, 0x8f, 0x76, + 0x5a, 0x07, 0x1c, 0xcd, 0xbe, 0xca, 0x16, 0x8a, 0x55, 0x30, 0xfe, 0x97, 0x81, 0xfc, 0x8e, 0x68, + 0x11, 0xc6, 0xbf, 0x75, 0x58, 0x30, 0xf1, 0x6f, 0x23, 0x4c, 0xd9, 0x44, 0xff, 0xd3, 0x26, 0xfb, + 0x5f, 0xba, 0xc2, 0xeb, 0x93, 0x15, 0x7e, 0x46, 0x21, 0xcc, 0xce, 0x2c, 0x84, 0xf7, 0x20, 0x1f, + 0xe2, 0x3e, 0xef, 0x82, 0x79, 0x95, 0x0f, 0x62, 0x84, 0xb6, 0xa7, 0x5a, 0xc4, 0xc2, 0xbc, 0x16, + 0xb1, 0x7f, 0x67, 0xa2, 0x49, 0xbc, 0xd3, 0x34, 0xf4, 0x69, 0xba, 0x60, 0x8a, 0x90, 0xef, 0x6b, + 0xa9, 0x92, 0xc9, 0x69, 0x71, 0xd5, 0x2c, 0xde, 0xa2, 0x6a, 0x6e, 0x2f, 0xc3, 0x92, 0x95, 0x76, + 0x6c, 0xbb, 0x02, 0x8b, 0x56, 0xc2, 0xe4, 0xab, 0x6c, 0x21, 0x53, 0xcd, 0x9a, 0xd3, 0x7d, 0xde, + 0xf8, 0x83, 0x06, 0x05, 0x13, 0xd3, 0x21, 0x09, 0x28, 0xe6, 0x79, 0x33, 0x8e, 0x8e, 0x15, 0x46, + 0xc1, 0x38, 0xcc, 0x4b, 0x34, 0x99, 0xb3, 0x6d, 0x17, 0x75, 0xe0, 0xde, 0x30, 0xc4, 0xd4, 0xeb, + 0x07, 0xd8, 0xb5, 0x86, 0x84, 0x32, 0x2b, 0x94, 0xb7, 0xa4, 0x5e, 0x55, 0xe3, 0xb6, 0x7c, 0x12, + 0xd3, 0x4e, 0x08, 0x65, 0xea, 0x2a, 0xcd, 0x95, 0xe1, 0x0c, 0xd4, 0xf8, 0x8b, 0x06, 0x1f, 0xec, + 0x90, 0xe0, 0xcc, 0x0b, 0x07, 0x1d, 0x12, 0x85, 0x0e, 0x3e, 0x1d, 0xfa, 0xc4, 0x76, 0x8d, 0xdf, + 0xdd, 0x5a, 0x03, 0x33, 0x8f, 0xa0, 0xcf, 0x3e, 0xc2, 0x53, 0x58, 0x72, 0x88, 0x8b, 0x2d, 0xcc, + 0xdf, 0x89, 0x43, 0xe2, 0x05, 0x4c, 0x75, 0x87, 0x0a, 0x87, 0x5b, 0x23, 0xd4, 0x80, 0x71, 0x8c, + 0x8c, 0x3f, 0x6a, 0x90, 0xd9, 0xc3, 0xcc, 0xe8, 0xbe, 0x0f, 0x97, 0x8c, 0xef, 0x27, 0x6e, 0x63, + 0x1d, 0x32, 0x61, 0x24, 0xdf, 0x97, 0x29, 0x8d, 0x25, 0x97, 0x98, 0x9c, 0x62, 0xfc, 0x53, 0x87, + 0xec, 0x81, 0x47, 0x99, 0xf1, 0x0f, 0xed, 0xd6, 0x5e, 0xbd, 0x98, 0x68, 0xa9, 0xf3, 0x1f, 0x8b, + 0xfb, 0x77, 0xe2, 0x86, 0xca, 0xd5, 0xf9, 0x63, 0x80, 0xa1, 0xdd, 0xc7, 0x16, 0x23, 0x6f, 0x70, + 0x20, 0x62, 0x56, 0x6a, 0xd4, 0x46, 0xeb, 0xbb, 0x1c, 0x3d, 0xb1, 0xfb, 0x5e, 0x20, 0x8c, 0xec, + 0x6b, 0x66, 0x91, 0xb3, 0x05, 0xfc, 0x4e, 0xd3, 0xb6, 0x8b, 0xb0, 0x60, 0x49, 0x5b, 0xdb, 0x65, + 0x28, 0x59, 0x63, 0x4b, 0xc6, 0xdb, 0x94, 0x16, 0xb3, 0x61, 0x14, 0xf0, 0xca, 0x9b, 0x99, 0x73, + 0x7c, 0xc1, 0x41, 0x3f, 0x85, 0xa5, 0x80, 0x77, 0x86, 0x84, 0x53, 0xfa, 0x7c, 0xa7, 0xcc, 0x32, + 0x5f, 0x70, 0x12, 0xbb, 0x65, 0x5c, 0x40, 0x7e, 0xc7, 0x0e, 0x1c, 0xec, 0xbf, 0xa7, 0x7b, 0x4d, + 0x2a, 0xe8, 0xfd, 0x36, 0xb4, 0xb5, 0xaf, 0xb5, 0x51, 0xcd, 0xbe, 0x0f, 0x77, 0xd3, 0x35, 0xdb, + 0x3a, 0x3d, 0x39, 0x38, 0x6e, 0xee, 0x56, 0xef, 0xa0, 0xbb, 0xb0, 0xac, 0xa6, 0xf6, 0x5a, 0x47, + 0x2d, 0xb3, 0xd9, 0x95, 0xd5, 0x7b, 0xba, 0xa2, 0xeb, 0xe8, 0x1e, 0x20, 0x85, 0x75, 0x4e, 0x0f, + 0x0f, 0x9b, 0x66, 0xfb, 0x35, 0xc7, 0x33, 0x33, 0x2b, 0x7d, 0x76, 0xba, 0xd2, 0xe7, 0x66, 0x56, + 0xfa, 0xfc, 0xda, 0xd7, 0x19, 0x28, 0xc4, 0xf5, 0x72, 0xfc, 0x78, 0xd2, 0x92, 0x8f, 0xa7, 0xc9, + 0xb7, 0x9b, 0x3e, 0xe3, 0xed, 0x36, 0xfb, 0x95, 0x96, 0xb9, 0xee, 0x95, 0xb6, 0xa1, 0xde, 0x63, + 0x59, 0x21, 0xab, 0x07, 0x53, 0x95, 0xbb, 0xde, 0xb5, 0xfb, 0x54, 0x7e, 0x59, 0xca, 0xc7, 0x9a, + 0x01, 0x85, 0x88, 0xe2, 0xd0, 0xb5, 0x99, 0xad, 0xde, 0x7e, 0xa3, 0xb1, 0xf1, 0x57, 0x0d, 0x96, + 0x65, 0x9b, 0x7a, 0x19, 0x92, 0x41, 0x47, 0x7e, 0xf3, 0x1b, 0xbf, 0xba, 0xb5, 0x82, 0x12, 0x2f, + 0x2f, 0x3d, 0xf5, 0xf2, 0x1a, 0x77, 0xa0, 0x4c, 0xb2, 0x03, 0x19, 0x5b, 0x89, 0x04, 0xf9, 0x1c, + 0x0a, 0x71, 0xd1, 0x57, 0x35, 0x62, 0x79, 0xea, 0x34, 0xe6, 0x88, 0x62, 0x7c, 0x09, 0xc5, 0xd1, + 0xd1, 0x66, 0x7c, 0x13, 0xaf, 0x24, 0xbf, 0x89, 0x8b, 0x89, 0xef, 0xde, 0xb5, 0x2e, 0x94, 0x53, + 0x6d, 0x8d, 0x3f, 0x69, 0x45, 0x83, 0x95, 0xab, 0xc5, 0x6f, 0xb4, 0x01, 0xc5, 0x78, 0x27, 0x7e, + 0x4d, 0x99, 0xd9, 0xde, 0x8c, 0x39, 0x6b, 0x7f, 0xd7, 0x00, 0x8d, 0x53, 0x79, 0xd7, 0xa3, 0x43, + 0x9b, 0x39, 0xe7, 0xdf, 0xa8, 0x03, 0xdd, 0x85, 0x3c, 0xff, 0x76, 0x1a, 0x05, 0x2f, 0x77, 0x41, + 0x7a, 0x6d, 0x37, 0x15, 0x97, 0xcc, 0x8d, 0x71, 0x19, 0x75, 0xda, 0xec, 0x2d, 0x3a, 0xed, 0x67, + 0x67, 0x50, 0x49, 0xe3, 0xe8, 0x63, 0x78, 0xd0, 0x69, 0x1f, 0x9e, 0x1e, 0x34, 0xbb, 0xed, 0xe3, + 0x23, 0xeb, 0xf0, 0x78, 0xb7, 0x65, 0x9d, 0x1e, 0x75, 0x4e, 0x5a, 0x3b, 0xed, 0x97, 0xed, 0x16, + 0x4f, 0xb4, 0x1a, 0xac, 0x4c, 0x12, 0xba, 0xad, 0x5f, 0x76, 0xab, 0x9a, 0xc8, 0xce, 0x89, 0x99, + 0xe6, 0xe9, 0x6e, 0xfb, 0xb8, 0xaa, 0x37, 0xfe, 0x9c, 0x83, 0xa5, 0x26, 0xd7, 0xee, 0x78, 0x37, + 0xe4, 0xc2, 0x07, 0x52, 0x6b, 0xe9, 0xff, 0x08, 0x3e, 0xbd, 0xa6, 0x6e, 0x4b, 0x6e, 0x3d, 0xee, + 0xac, 0xdf, 0xb9, 0x89, 0xa6, 0x54, 0xf5, 0x4e, 0x83, 0x87, 0x71, 0x07, 0x1e, 0x11, 0x93, 0xbd, + 0x18, 0x35, 0xae, 0xb3, 0x34, 0xdd, 0xb7, 0x47, 0xbb, 0x7f, 0xf1, 0x8d, 0xd6, 0x28, 0x57, 0x7e, + 0x0d, 0xd5, 0x3d, 0xcc, 0xd2, 0xa7, 0x5d, 0xbb, 0xc6, 0xd0, 0x1e, 0x66, 0xa3, 0xcd, 0x9e, 0xcc, + 0xe5, 0x28, 0xe3, 0x16, 0x20, 0xde, 0x31, 0x53, 0x0c, 0x8a, 0xae, 0x5b, 0xca, 0xa9, 0x23, 0xfb, + 0x9f, 0xcc, 0x27, 0xa9, 0x0d, 0xf8, 0x75, 0x89, 0x8e, 0x72, 0xcb, 0xeb, 0x12, 0xdc, 0x9b, 0xaf, + 0x2b, 0xa6, 0xa9, 0x5d, 0x2e, 0xe1, 0xbe, 0x12, 0x85, 0xd2, 0x73, 0xa2, 0x10, 0xa1, 0x67, 0xd3, + 0xd5, 0x6d, 0xaa, 0x5a, 0x8d, 0x76, 0xfc, 0xde, 0xed, 0xc8, 0x72, 0xdf, 0xed, 0x97, 0xaf, 0x9f, + 0xf4, 0x3d, 0x76, 0x1e, 0xf5, 0xea, 0x0e, 0x19, 0xc4, 0xff, 0x7d, 0xca, 0xff, 0x37, 0x1d, 0xe2, + 0xc7, 0xc0, 0xdf, 0xf4, 0xf2, 0x81, 0x77, 0x89, 0x7f, 0x26, 0x1e, 0x81, 0x84, 0x91, 0xff, 0xe8, + 0x15, 0x35, 0x7e, 0xf1, 0x42, 0x00, 0xbd, 0xbc, 0x58, 0xf2, 0xc5, 0xff, 0x03, 0x00, 0x00, 0xff, + 0xff, 0xca, 0xe3, 0x43, 0x02, 0x6e, 0x15, 0x00, 0x00, } From db02109bb0effee7da000d6fe016f49ae0888270 Mon Sep 17 00:00:00 2001 From: Arseny Boykov Date: Thu, 25 Jun 2026 02:34:21 +0400 Subject: [PATCH 3/5] proto(agent-sim): add per-turn breakdown to SimulationRun.Job.AudioMetrics Adds a repeated TurnMetrics message under Job.AudioMetrics carrying the per-turn scalars (response latency, WER, STT/LLM/TTS timings, e2e) behind the existing call-level aggregates. Co-Authored-By: Claude Opus 4.8 --- livekit/livekit_agent_simulation.pb.go | 291 ++++++++++++++++------- protobufs/livekit_agent_simulation.proto | 16 ++ 2 files changed, 221 insertions(+), 86 deletions(-) diff --git a/livekit/livekit_agent_simulation.pb.go b/livekit/livekit_agent_simulation.pb.go index 54705f3bd..28e6de167 100644 --- a/livekit/livekit_agent_simulation.pb.go +++ b/livekit/livekit_agent_simulation.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v7.35.1 +// protoc v7.35.0 // source: livekit_agent_simulation.proto package livekit @@ -1177,7 +1177,10 @@ type SimulationRun_Job_AudioMetrics struct { // Call-level barge-in count. Interruptions int32 `protobuf:"varint,7,opt,name=interruptions,proto3" json:"interruptions,omitempty"` // Turns that contributed measurements (context for the aggregates). - SampleCount int32 `protobuf:"varint,8,opt,name=sample_count,json=sampleCount,proto3" json:"sample_count,omitempty"` + SampleCount int32 `protobuf:"varint,8,opt,name=sample_count,json=sampleCount,proto3" json:"sample_count,omitempty"` + // Per-turn breakdown behind the aggregates above, one entry per measured + // turn in order. A -1 field means the datum was unavailable for that turn. + Turns []*SimulationRun_Job_AudioMetrics_TurnMetrics `protobuf:"bytes,9,rep,name=turns,proto3" json:"turns,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1268,6 +1271,108 @@ func (x *SimulationRun_Job_AudioMetrics) GetSampleCount() int32 { return 0 } +func (x *SimulationRun_Job_AudioMetrics) GetTurns() []*SimulationRun_Job_AudioMetrics_TurnMetrics { + if x != nil { + return x.Turns + } + return nil +} + +type SimulationRun_Job_AudioMetrics_TurnMetrics struct { + state protoimpl.MessageState `protogen:"open.v1"` + Turn int32 `protobuf:"varint,1,opt,name=turn,proto3" json:"turn,omitempty"` // 1-based turn index + // Tier 0 — true end-to-end response latency off the waveform. + ResponseLatencyS float32 `protobuf:"fixed32,2,opt,name=response_latency_s,json=responseLatencyS,proto3" json:"response_latency_s,omitempty"` + // Tier 1 — the agent's own pipeline, broadcast over its RemoteSession; + // -1 when the backchannel is absent. + Wer float32 `protobuf:"fixed32,3,opt,name=wer,proto3" json:"wer,omitempty"` + SttDelayS float32 `protobuf:"fixed32,4,opt,name=stt_delay_s,json=sttDelayS,proto3" json:"stt_delay_s,omitempty"` + LlmTtftS float32 `protobuf:"fixed32,5,opt,name=llm_ttft_s,json=llmTtftS,proto3" json:"llm_ttft_s,omitempty"` + TtsTtfbS float32 `protobuf:"fixed32,6,opt,name=tts_ttfb_s,json=ttsTtfbS,proto3" json:"tts_ttfb_s,omitempty"` + E2ELatencyS float32 `protobuf:"fixed32,7,opt,name=e2e_latency_s,json=e2eLatencyS,proto3" json:"e2e_latency_s,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) Reset() { + *x = SimulationRun_Job_AudioMetrics_TurnMetrics{} + mi := &file_livekit_agent_simulation_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SimulationRun_Job_AudioMetrics_TurnMetrics) ProtoMessage() {} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) ProtoReflect() protoreflect.Message { + mi := &file_livekit_agent_simulation_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SimulationRun_Job_AudioMetrics_TurnMetrics.ProtoReflect.Descriptor instead. +func (*SimulationRun_Job_AudioMetrics_TurnMetrics) Descriptor() ([]byte, []int) { + return file_livekit_agent_simulation_proto_rawDescGZIP(), []int{1, 0, 1, 0} +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetTurn() int32 { + if x != nil { + return x.Turn + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetResponseLatencyS() float32 { + if x != nil { + return x.ResponseLatencyS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetWer() float32 { + if x != nil { + return x.Wer + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetSttDelayS() float32 { + if x != nil { + return x.SttDelayS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetLlmTtftS() float32 { + if x != nil { + return x.LlmTtftS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetTtsTtfbS() float32 { + if x != nil { + return x.TtsTtfbS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetE2ELatencyS() float32 { + if x != nil { + return x.E2ELatencyS + } + return 0 +} + type SimulationRun_Create_Request struct { state protoimpl.MessageState `protogen:"open.v1"` ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` @@ -1288,7 +1393,7 @@ type SimulationRun_Create_Request struct { func (x *SimulationRun_Create_Request) Reset() { *x = SimulationRun_Create_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[16] + mi := &file_livekit_agent_simulation_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1300,7 +1405,7 @@ func (x *SimulationRun_Create_Request) String() string { func (*SimulationRun_Create_Request) ProtoMessage() {} func (x *SimulationRun_Create_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[16] + mi := &file_livekit_agent_simulation_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1375,7 +1480,7 @@ type SimulationRun_Create_Response struct { func (x *SimulationRun_Create_Response) Reset() { *x = SimulationRun_Create_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[17] + mi := &file_livekit_agent_simulation_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1387,7 +1492,7 @@ func (x *SimulationRun_Create_Response) String() string { func (*SimulationRun_Create_Response) ProtoMessage() {} func (x *SimulationRun_Create_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[17] + mi := &file_livekit_agent_simulation_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1428,7 +1533,7 @@ type SimulationRun_ConfirmSourceUpload_Request struct { func (x *SimulationRun_ConfirmSourceUpload_Request) Reset() { *x = SimulationRun_ConfirmSourceUpload_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[18] + mi := &file_livekit_agent_simulation_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1440,7 +1545,7 @@ func (x *SimulationRun_ConfirmSourceUpload_Request) String() string { func (*SimulationRun_ConfirmSourceUpload_Request) ProtoMessage() {} func (x *SimulationRun_ConfirmSourceUpload_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[18] + mi := &file_livekit_agent_simulation_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1485,7 +1590,7 @@ type SimulationRun_ConfirmSourceUpload_Response struct { func (x *SimulationRun_ConfirmSourceUpload_Response) Reset() { *x = SimulationRun_ConfirmSourceUpload_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[19] + mi := &file_livekit_agent_simulation_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1497,7 +1602,7 @@ func (x *SimulationRun_ConfirmSourceUpload_Response) String() string { func (*SimulationRun_ConfirmSourceUpload_Response) ProtoMessage() {} func (x *SimulationRun_ConfirmSourceUpload_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[19] + mi := &file_livekit_agent_simulation_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1523,7 +1628,7 @@ type SimulationRun_Get_Request struct { func (x *SimulationRun_Get_Request) Reset() { *x = SimulationRun_Get_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[20] + mi := &file_livekit_agent_simulation_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1535,7 +1640,7 @@ func (x *SimulationRun_Get_Request) String() string { func (*SimulationRun_Get_Request) ProtoMessage() {} func (x *SimulationRun_Get_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[20] + mi := &file_livekit_agent_simulation_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1574,7 +1679,7 @@ type SimulationRun_Get_Response struct { func (x *SimulationRun_Get_Response) Reset() { *x = SimulationRun_Get_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[21] + mi := &file_livekit_agent_simulation_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1586,7 +1691,7 @@ func (x *SimulationRun_Get_Response) String() string { func (*SimulationRun_Get_Response) ProtoMessage() {} func (x *SimulationRun_Get_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[21] + mi := &file_livekit_agent_simulation_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1620,7 +1725,7 @@ type SimulationRun_List_Request struct { func (x *SimulationRun_List_Request) Reset() { *x = SimulationRun_List_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[22] + mi := &file_livekit_agent_simulation_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1632,7 +1737,7 @@ func (x *SimulationRun_List_Request) String() string { func (*SimulationRun_List_Request) ProtoMessage() {} func (x *SimulationRun_List_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[22] + mi := &file_livekit_agent_simulation_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1679,7 +1784,7 @@ type SimulationRun_List_Response struct { func (x *SimulationRun_List_Response) Reset() { *x = SimulationRun_List_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[23] + mi := &file_livekit_agent_simulation_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1691,7 +1796,7 @@ func (x *SimulationRun_List_Response) String() string { func (*SimulationRun_List_Response) ProtoMessage() {} func (x *SimulationRun_List_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[23] + mi := &file_livekit_agent_simulation_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1731,7 +1836,7 @@ type SimulationRun_Cancel_Request struct { func (x *SimulationRun_Cancel_Request) Reset() { *x = SimulationRun_Cancel_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[24] + mi := &file_livekit_agent_simulation_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1743,7 +1848,7 @@ func (x *SimulationRun_Cancel_Request) String() string { func (*SimulationRun_Cancel_Request) ProtoMessage() {} func (x *SimulationRun_Cancel_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[24] + mi := &file_livekit_agent_simulation_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1781,7 +1886,7 @@ type SimulationRun_Cancel_Response struct { func (x *SimulationRun_Cancel_Response) Reset() { *x = SimulationRun_Cancel_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[25] + mi := &file_livekit_agent_simulation_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1793,7 +1898,7 @@ func (x *SimulationRun_Cancel_Response) String() string { func (*SimulationRun_Cancel_Response) ProtoMessage() {} func (x *SimulationRun_Cancel_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[25] + mi := &file_livekit_agent_simulation_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1817,7 +1922,7 @@ type Scenario_CreateFromSession struct { func (x *Scenario_CreateFromSession) Reset() { *x = Scenario_CreateFromSession{} - mi := &file_livekit_agent_simulation_proto_msgTypes[26] + mi := &file_livekit_agent_simulation_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1829,7 +1934,7 @@ func (x *Scenario_CreateFromSession) String() string { func (*Scenario_CreateFromSession) ProtoMessage() {} func (x *Scenario_CreateFromSession) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[26] + mi := &file_livekit_agent_simulation_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1856,7 +1961,7 @@ type Scenario_CreateFromSession_Request struct { func (x *Scenario_CreateFromSession_Request) Reset() { *x = Scenario_CreateFromSession_Request{} - mi := &file_livekit_agent_simulation_proto_msgTypes[28] + mi := &file_livekit_agent_simulation_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1868,7 +1973,7 @@ func (x *Scenario_CreateFromSession_Request) String() string { func (*Scenario_CreateFromSession_Request) ProtoMessage() {} func (x *Scenario_CreateFromSession_Request) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[28] + mi := &file_livekit_agent_simulation_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1914,7 +2019,7 @@ type Scenario_CreateFromSession_Response struct { func (x *Scenario_CreateFromSession_Response) Reset() { *x = Scenario_CreateFromSession_Response{} - mi := &file_livekit_agent_simulation_proto_msgTypes[29] + mi := &file_livekit_agent_simulation_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1926,7 +2031,7 @@ func (x *Scenario_CreateFromSession_Response) String() string { func (*Scenario_CreateFromSession_Response) ProtoMessage() {} func (x *Scenario_CreateFromSession_Response) ProtoReflect() protoreflect.Message { - mi := &file_livekit_agent_simulation_proto_msgTypes[29] + mi := &file_livekit_agent_simulation_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1971,7 +2076,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\n" + "suggestion\x18\x02 \x01(\tR\n" + "suggestion\x12\x14\n" + - "\x05label\x18\x03 \x01(\tR\x05label\"\xec\x19\n" + + "\x05label\x18\x03 \x01(\tR\x05label\"\x9b\x1c\n" + "\rSimulationRun\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1d\n" + "\n" + @@ -1994,7 +2099,8 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x0fnum_simulations\x18\x0f \x01(\x05R\x0enumSimulations\x122\n" + "\x05usage\x18\x10 \x01(\v2\x1c.livekit.SimulationRun.UsageR\x05usage\x12 \n" + "\vconcurrency\x18\x11 \x01(\x05R\vconcurrency\x12+\n" + - "\x04mode\x18\x12 \x01(\x0e2\x17.livekit.SimulationModeR\x04mode\x1a\x85\b\n" + + "\x04mode\x18\x12 \x01(\x0e2\x17.livekit.SimulationModeR\x04mode\x1a\xb4\n" + + "\n" + "\x03Job\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x129\n" + "\x06status\x18\x02 \x01(\x0e2!.livekit.SimulationRun.Job.StatusR\x06status\x12\"\n" + @@ -2013,7 +2119,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\raudio_metrics\x18\x0e \x01(\v2'.livekit.SimulationRun.Job.AudioMetricsR\faudioMetrics\x1a]\n" + "\x05Usage\x12(\n" + "\x10text_turns_count\x18\x01 \x01(\x05R\x0etextTurnsCount\x12*\n" + - "\x11audio_turns_count\x18\x02 \x01(\x05R\x0faudioTurnsCount\x1a\x9b\x02\n" + + "\x11audio_turns_count\x18\x02 \x01(\x05R\x0faudioTurnsCount\x1a\xca\x04\n" + "\fAudioMetrics\x12)\n" + "\x11e2e_latency_p50_s\x18\x01 \x01(\x02R\x0ee2eLatencyP50S\x12)\n" + "\x11e2e_latency_p95_s\x18\x02 \x01(\x02R\x0ee2eLatencyP95S\x12\x10\n" + @@ -2024,7 +2130,18 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\n" + "tts_ttfb_s\x18\x06 \x01(\x02R\bttsTtfbS\x12$\n" + "\rinterruptions\x18\a \x01(\x05R\rinterruptions\x12!\n" + - "\fsample_count\x18\b \x01(\x05R\vsampleCount\"o\n" + + "\fsample_count\x18\b \x01(\x05R\vsampleCount\x12I\n" + + "\x05turns\x18\t \x03(\v23.livekit.SimulationRun.Job.AudioMetrics.TurnMetricsR\x05turns\x1a\xe1\x01\n" + + "\vTurnMetrics\x12\x12\n" + + "\x04turn\x18\x01 \x01(\x05R\x04turn\x12,\n" + + "\x12response_latency_s\x18\x02 \x01(\x02R\x10responseLatencyS\x12\x10\n" + + "\x03wer\x18\x03 \x01(\x02R\x03wer\x12\x1e\n" + + "\vstt_delay_s\x18\x04 \x01(\x02R\tsttDelayS\x12\x1c\n" + + "\n" + + "llm_ttft_s\x18\x05 \x01(\x02R\bllmTtftS\x12\x1c\n" + + "\n" + + "tts_ttfb_s\x18\x06 \x01(\x02R\bttsTtfbS\x12\"\n" + + "\re2e_latency_s\x18\a \x01(\x02R\ve2eLatencyS\"o\n" + "\x06Status\x12\x12\n" + "\x0eSTATUS_PENDING\x10\x00\x12\x12\n" + "\x0eSTATUS_RUNNING\x10\x01\x12\x14\n" + @@ -2143,7 +2260,7 @@ func file_livekit_agent_simulation_proto_rawDescGZIP() []byte { } var file_livekit_agent_simulation_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_livekit_agent_simulation_proto_msgTypes = make([]protoimpl.MessageInfo, 30) +var file_livekit_agent_simulation_proto_msgTypes = make([]protoimpl.MessageInfo, 31) var file_livekit_agent_simulation_proto_goTypes = []any{ (SimulationMode)(0), // 0: livekit.SimulationMode (SimulationRun_Status)(0), // 1: livekit.SimulationRun.Status @@ -2164,72 +2281,74 @@ var file_livekit_agent_simulation_proto_goTypes = []any{ (*SimulationRun_Usage)(nil), // 16: livekit.SimulationRun.Usage (*SimulationRun_Job_Usage)(nil), // 17: livekit.SimulationRun.Job.Usage (*SimulationRun_Job_AudioMetrics)(nil), // 18: livekit.SimulationRun.Job.AudioMetrics - (*SimulationRun_Create_Request)(nil), // 19: livekit.SimulationRun.Create.Request - (*SimulationRun_Create_Response)(nil), // 20: livekit.SimulationRun.Create.Response - (*SimulationRun_ConfirmSourceUpload_Request)(nil), // 21: livekit.SimulationRun.ConfirmSourceUpload.Request - (*SimulationRun_ConfirmSourceUpload_Response)(nil), // 22: livekit.SimulationRun.ConfirmSourceUpload.Response - (*SimulationRun_Get_Request)(nil), // 23: livekit.SimulationRun.Get.Request - (*SimulationRun_Get_Response)(nil), // 24: livekit.SimulationRun.Get.Response - (*SimulationRun_List_Request)(nil), // 25: livekit.SimulationRun.List.Request - (*SimulationRun_List_Response)(nil), // 26: livekit.SimulationRun.List.Response - (*SimulationRun_Cancel_Request)(nil), // 27: livekit.SimulationRun.Cancel.Request - (*SimulationRun_Cancel_Response)(nil), // 28: livekit.SimulationRun.Cancel.Response - (*Scenario_CreateFromSession)(nil), // 29: livekit.Scenario.CreateFromSession - nil, // 30: livekit.Scenario.TagsEntry - (*Scenario_CreateFromSession_Request)(nil), // 31: livekit.Scenario.CreateFromSession.Request - (*Scenario_CreateFromSession_Response)(nil), // 32: livekit.Scenario.CreateFromSession.Response - (*timestamppb.Timestamp)(nil), // 33: google.protobuf.Timestamp - (*agent.ChatContext)(nil), // 34: livekit.agent.ChatContext - (*PresignedPostRequest)(nil), // 35: livekit.PresignedPostRequest - (*TokenPagination)(nil), // 36: livekit.TokenPagination + (*SimulationRun_Job_AudioMetrics_TurnMetrics)(nil), // 19: livekit.SimulationRun.Job.AudioMetrics.TurnMetrics + (*SimulationRun_Create_Request)(nil), // 20: livekit.SimulationRun.Create.Request + (*SimulationRun_Create_Response)(nil), // 21: livekit.SimulationRun.Create.Response + (*SimulationRun_ConfirmSourceUpload_Request)(nil), // 22: livekit.SimulationRun.ConfirmSourceUpload.Request + (*SimulationRun_ConfirmSourceUpload_Response)(nil), // 23: livekit.SimulationRun.ConfirmSourceUpload.Response + (*SimulationRun_Get_Request)(nil), // 24: livekit.SimulationRun.Get.Request + (*SimulationRun_Get_Response)(nil), // 25: livekit.SimulationRun.Get.Response + (*SimulationRun_List_Request)(nil), // 26: livekit.SimulationRun.List.Request + (*SimulationRun_List_Response)(nil), // 27: livekit.SimulationRun.List.Response + (*SimulationRun_Cancel_Request)(nil), // 28: livekit.SimulationRun.Cancel.Request + (*SimulationRun_Cancel_Response)(nil), // 29: livekit.SimulationRun.Cancel.Response + (*Scenario_CreateFromSession)(nil), // 30: livekit.Scenario.CreateFromSession + nil, // 31: livekit.Scenario.TagsEntry + (*Scenario_CreateFromSession_Request)(nil), // 32: livekit.Scenario.CreateFromSession.Request + (*Scenario_CreateFromSession_Response)(nil), // 33: livekit.Scenario.CreateFromSession.Response + (*timestamppb.Timestamp)(nil), // 34: google.protobuf.Timestamp + (*agent.ChatContext)(nil), // 35: livekit.agent.ChatContext + (*PresignedPostRequest)(nil), // 36: livekit.PresignedPostRequest + (*TokenPagination)(nil), // 37: livekit.TokenPagination } var file_livekit_agent_simulation_proto_depIdxs = []int32{ 9, // 0: livekit.SimulationRunSummary.issues:type_name -> livekit.SimulationRunSummary.Issue 8, // 1: livekit.SimulationRunSummary.chat_history:type_name -> livekit.SimulationRunSummary.ChatHistoryEntry 1, // 2: livekit.SimulationRun.status:type_name -> livekit.SimulationRun.Status - 33, // 3: livekit.SimulationRun.created_at:type_name -> google.protobuf.Timestamp + 34, // 3: livekit.SimulationRun.created_at:type_name -> google.protobuf.Timestamp 10, // 4: livekit.SimulationRun.jobs:type_name -> livekit.SimulationRun.Job 3, // 5: livekit.SimulationRun.summary:type_name -> livekit.SimulationRunSummary 6, // 6: livekit.SimulationRun.scenario_group:type_name -> livekit.ScenarioGroup - 33, // 7: livekit.SimulationRun.ended_at:type_name -> google.protobuf.Timestamp + 34, // 7: livekit.SimulationRun.ended_at:type_name -> google.protobuf.Timestamp 16, // 8: livekit.SimulationRun.usage:type_name -> livekit.SimulationRun.Usage 0, // 9: livekit.SimulationRun.mode:type_name -> livekit.SimulationMode - 30, // 10: livekit.Scenario.tags:type_name -> livekit.Scenario.TagsEntry + 31, // 10: livekit.Scenario.tags:type_name -> livekit.Scenario.TagsEntry 5, // 11: livekit.ScenarioGroup.scenarios:type_name -> livekit.Scenario 5, // 12: livekit.SimulationDispatch.scenario:type_name -> livekit.Scenario 0, // 13: livekit.SimulationDispatch.mode:type_name -> livekit.SimulationMode - 34, // 14: livekit.SimulationRunSummary.ChatHistoryEntry.value:type_name -> livekit.agent.ChatContext + 35, // 14: livekit.SimulationRunSummary.ChatHistoryEntry.value:type_name -> livekit.agent.ChatContext 2, // 15: livekit.SimulationRun.Job.status:type_name -> livekit.SimulationRun.Job.Status - 33, // 16: livekit.SimulationRun.Job.started_at:type_name -> google.protobuf.Timestamp - 33, // 17: livekit.SimulationRun.Job.ended_at:type_name -> google.protobuf.Timestamp + 34, // 16: livekit.SimulationRun.Job.started_at:type_name -> google.protobuf.Timestamp + 34, // 17: livekit.SimulationRun.Job.ended_at:type_name -> google.protobuf.Timestamp 17, // 18: livekit.SimulationRun.Job.usage:type_name -> livekit.SimulationRun.Job.Usage 18, // 19: livekit.SimulationRun.Job.audio_metrics:type_name -> livekit.SimulationRun.Job.AudioMetrics - 6, // 20: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup - 0, // 21: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode - 35, // 22: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest - 4, // 23: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun - 1, // 24: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status - 36, // 25: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination - 4, // 26: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun - 36, // 27: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination - 5, // 28: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario - 19, // 29: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request - 21, // 30: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request - 23, // 31: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request - 25, // 32: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request - 27, // 33: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request - 31, // 34: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request - 20, // 35: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response - 22, // 36: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response - 24, // 37: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response - 26, // 38: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response - 28, // 39: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response - 32, // 40: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response - 35, // [35:41] is the sub-list for method output_type - 29, // [29:35] is the sub-list for method input_type - 29, // [29:29] is the sub-list for extension type_name - 29, // [29:29] is the sub-list for extension extendee - 0, // [0:29] is the sub-list for field type_name + 19, // 20: livekit.SimulationRun.Job.AudioMetrics.turns:type_name -> livekit.SimulationRun.Job.AudioMetrics.TurnMetrics + 6, // 21: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup + 0, // 22: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode + 36, // 23: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest + 4, // 24: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun + 1, // 25: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status + 37, // 26: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination + 4, // 27: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun + 37, // 28: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination + 5, // 29: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario + 20, // 30: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request + 22, // 31: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request + 24, // 32: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request + 26, // 33: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request + 28, // 34: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request + 32, // 35: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request + 21, // 36: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response + 23, // 37: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response + 25, // 38: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response + 27, // 39: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response + 29, // 40: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response + 33, // 41: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response + 36, // [36:42] is the sub-list for method output_type + 30, // [30:36] is the sub-list for method input_type + 30, // [30:30] is the sub-list for extension type_name + 30, // [30:30] is the sub-list for extension extendee + 0, // [0:30] is the sub-list for field type_name } func init() { file_livekit_agent_simulation_proto_init() } @@ -2239,15 +2358,15 @@ func file_livekit_agent_simulation_proto_init() { } file_livekit_cloud_agent_proto_init() file_livekit_models_proto_init() - file_livekit_agent_simulation_proto_msgTypes[16].OneofWrappers = []any{} - file_livekit_agent_simulation_proto_msgTypes[22].OneofWrappers = []any{} + file_livekit_agent_simulation_proto_msgTypes[17].OneofWrappers = []any{} + file_livekit_agent_simulation_proto_msgTypes[23].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_livekit_agent_simulation_proto_rawDesc), len(file_livekit_agent_simulation_proto_rawDesc)), NumEnums: 3, - NumMessages: 30, + NumMessages: 31, NumExtensions: 0, NumServices: 1, }, diff --git a/protobufs/livekit_agent_simulation.proto b/protobufs/livekit_agent_simulation.proto index ad4303dec..8fbd1a939 100644 --- a/protobufs/livekit_agent_simulation.proto +++ b/protobufs/livekit_agent_simulation.proto @@ -96,6 +96,22 @@ message SimulationRun { int32 interruptions = 7; // Turns that contributed measurements (context for the aggregates). int32 sample_count = 8; + // Per-turn breakdown behind the aggregates above, one entry per measured + // turn in order. A -1 field means the datum was unavailable for that turn. + repeated TurnMetrics turns = 9; + + message TurnMetrics { + int32 turn = 1; // 1-based turn index + // Tier 0 — true end-to-end response latency off the waveform. + float response_latency_s = 2; + // Tier 1 — the agent's own pipeline, broadcast over its RemoteSession; + // -1 when the backchannel is absent. + float wer = 3; + float stt_delay_s = 4; + float llm_ttft_s = 5; + float tts_ttfb_s = 6; + float e2e_latency_s = 7; + } } } From abd3ba7c3151b4acef77300160b5d0f0c424977b Mon Sep 17 00:00:00 2001 From: Arseny Boykov Date: Wed, 1 Jul 2026 16:28:09 +0400 Subject: [PATCH 4/5] Update SimulationRun.Job.AudioMetrics --- livekit/livekit_agent_simulation.pb.go | 495 +++++++++++++++++++---- protobufs/livekit_agent_simulation.proto | 105 +++-- 2 files changed, 497 insertions(+), 103 deletions(-) diff --git a/livekit/livekit_agent_simulation.pb.go b/livekit/livekit_agent_simulation.pb.go index 28e6de167..14169f16e 100644 --- a/livekit/livekit_agent_simulation.pb.go +++ b/livekit/livekit_agent_simulation.pb.go @@ -750,9 +750,8 @@ type SimulationRun_Job struct { EndedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` RoomId string `protobuf:"bytes,12,opt,name=room_id,json=roomId,proto3" json:"room_id,omitempty"` Usage *SimulationRun_Job_Usage `protobuf:"bytes,13,opt,name=usage,proto3" json:"usage,omitempty"` - // Audio-mode pipeline metrics aggregated over the call's measured turns. - // Only populated for audio jobs. A -1 field means the datum was - // unavailable (Tier-1 metrics require the agent's backchannel). + // Metrics for an audio job, aggregated over the call's measured turns. + // Unset for text jobs; a -1 field was not measured (see AudioMetrics). AudioMetrics *SimulationRun_Job_AudioMetrics `protobuf:"bytes,14,opt,name=audio_metrics,json=audioMetrics,proto3" json:"audio_metrics,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1163,26 +1162,65 @@ func (x *SimulationRun_Job_Usage) GetAudioTurnsCount() int32 { return 0 } +// Per-call audio metrics, grouped by pipeline stage. A -1 means the metric +// was not measured for this call; for counts, -1 = not measured and 0 = a +// measured zero. Scores are 0-1 unless noted. type SimulationRun_Job_AudioMetrics struct { state protoimpl.MessageState `protogen:"open.v1"` - // Tier 0 — true end-to-end response latency off the waveform (our - // playout end -> the agent's speech onset). Always present for audio. - E2ELatencyP50S float32 `protobuf:"fixed32,1,opt,name=e2e_latency_p50_s,json=e2eLatencyP50S,proto3" json:"e2e_latency_p50_s,omitempty"` - E2ELatencyP95S float32 `protobuf:"fixed32,2,opt,name=e2e_latency_p95_s,json=e2eLatencyP95S,proto3" json:"e2e_latency_p95_s,omitempty"` - // Tier 1 — mean over turns; -1 when the backchannel is absent. - Wer float32 `protobuf:"fixed32,3,opt,name=wer,proto3" json:"wer,omitempty"` // normalized word error rate of the agent's STT of us - LlmTtftS float32 `protobuf:"fixed32,4,opt,name=llm_ttft_s,json=llmTtftS,proto3" json:"llm_ttft_s,omitempty"` - SttDelayS float32 `protobuf:"fixed32,5,opt,name=stt_delay_s,json=sttDelayS,proto3" json:"stt_delay_s,omitempty"` - TtsTtfbS float32 `protobuf:"fixed32,6,opt,name=tts_ttfb_s,json=ttsTtfbS,proto3" json:"tts_ttfb_s,omitempty"` - // Call-level barge-in count. - Interruptions int32 `protobuf:"varint,7,opt,name=interruptions,proto3" json:"interruptions,omitempty"` - // Turns that contributed measurements (context for the aggregates). - SampleCount int32 `protobuf:"varint,8,opt,name=sample_count,json=sampleCount,proto3" json:"sample_count,omitempty"` - // Per-turn breakdown behind the aggregates above, one entry per measured - // turn in order. A -1 field means the datum was unavailable for that turn. - Turns []*SimulationRun_Job_AudioMetrics_TurnMetrics `protobuf:"bytes,9,rep,name=turns,proto3" json:"turns,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // -- overall -- + AccuracyScore float32 `protobuf:"fixed32,1,opt,name=accuracy_score,json=accuracyScore,proto3" json:"accuracy_score,omitempty"` // how accurately the agent achieved the intended outcome + ExperienceScore float32 `protobuf:"fixed32,2,opt,name=experience_score,json=experienceScore,proto3" json:"experience_score,omitempty"` // overall quality of the caller experience + // -- speech recognition -- + SttWer float32 `protobuf:"fixed32,33,opt,name=stt_wer,json=sttWer,proto3" json:"stt_wer,omitempty"` // word error rate of the agent's transcription of the caller + SttCer float32 `protobuf:"fixed32,34,opt,name=stt_cer,json=sttCer,proto3" json:"stt_cer,omitempty"` // character error rate + SttKeytermAccuracy float32 `protobuf:"fixed32,35,opt,name=stt_keyterm_accuracy,json=sttKeytermAccuracy,proto3" json:"stt_keyterm_accuracy,omitempty"` // recall of key terms (names, IDs, amounts) in the transcription + SttTranscriptionLatencyS float32 `protobuf:"fixed32,40,opt,name=stt_transcription_latency_s,json=sttTranscriptionLatencyS,proto3" json:"stt_transcription_latency_s,omitempty"` // seconds from the end of caller speech to the final transcript + // -- speech synthesis -- + TtsTtfbS float32 `protobuf:"fixed32,43,opt,name=tts_ttfb_s,json=ttsTtfbS,proto3" json:"tts_ttfb_s,omitempty"` // seconds to the first audio of a reply + TtsSpeechRateWpm float32 `protobuf:"fixed32,45,opt,name=tts_speech_rate_wpm,json=ttsSpeechRateWpm,proto3" json:"tts_speech_rate_wpm,omitempty"` // speaking rate, words per minute + TtsNaturalnessScore float32 `protobuf:"fixed32,22,opt,name=tts_naturalness_score,json=ttsNaturalnessScore,proto3" json:"tts_naturalness_score,omitempty"` // overall perceptual naturalness + TtsProsodyScore float32 `protobuf:"fixed32,37,opt,name=tts_prosody_score,json=ttsProsodyScore,proto3" json:"tts_prosody_score,omitempty"` // pitch and intonation + TtsExpressivenessScore float32 `protobuf:"fixed32,38,opt,name=tts_expressiveness_score,json=ttsExpressivenessScore,proto3" json:"tts_expressiveness_score,omitempty"` // expressiveness and affect + TtsEnunciationScore float32 `protobuf:"fixed32,32,opt,name=tts_enunciation_score,json=ttsEnunciationScore,proto3" json:"tts_enunciation_score,omitempty"` // clarity of structured entities (emails, phone numbers, codes) + TtsUptalkScore float32 `protobuf:"fixed32,39,opt,name=tts_uptalk_score,json=ttsUptalkScore,proto3" json:"tts_uptalk_score,omitempty"` // appropriate terminal intonation; low = inappropriate uptalk + TtsWer float32 `protobuf:"fixed32,36,opt,name=tts_wer,json=ttsWer,proto3" json:"tts_wer,omitempty"` // intelligibility: intended text vs a transcription of the output audio + TtsConversationalityScore float32 `protobuf:"fixed32,55,opt,name=tts_conversationality_score,json=ttsConversationalityScore,proto3" json:"tts_conversationality_score,omitempty"` // sounds conversational rather than read-aloud + TtsHallucinationRate float32 `protobuf:"fixed32,56,opt,name=tts_hallucination_rate,json=ttsHallucinationRate,proto3" json:"tts_hallucination_rate,omitempty"` // fraction of output audio not traceable to the input text + TtsAudioQualityMos float32 `protobuf:"fixed32,57,opt,name=tts_audio_quality_mos,json=ttsAudioQualityMos,proto3" json:"tts_audio_quality_mos,omitempty"` // 1-5 mean opinion score (coloration, noise, discontinuity, loudness) + TtsVoiceConsistencyScore float32 `protobuf:"fixed32,58,opt,name=tts_voice_consistency_score,json=ttsVoiceConsistencyScore,proto3" json:"tts_voice_consistency_score,omitempty"` // voice stays consistent within and across turns + TtsSpokenArtifactCount int32 `protobuf:"varint,59,opt,name=tts_spoken_artifact_count,json=ttsSpokenArtifactCount,proto3" json:"tts_spoken_artifact_count,omitempty"` // spoken markup, clicks, or vowel prolongation + // -- language model -- + LlmTtftS float32 `protobuf:"fixed32,41,opt,name=llm_ttft_s,json=llmTtftS,proto3" json:"llm_ttft_s,omitempty"` // seconds to the first token + LlmTtfsS float32 `protobuf:"fixed32,48,opt,name=llm_ttfs_s,json=llmTtfsS,proto3" json:"llm_ttfs_s,omitempty"` // seconds to the first complete sentence + LlmThroughputTps float32 `protobuf:"fixed32,42,opt,name=llm_throughput_tps,json=llmThroughputTps,proto3" json:"llm_throughput_tps,omitempty"` // output tokens per second + // -- conversation timing & turn-taking -- + TurnTakingScore float32 `protobuf:"fixed32,10,opt,name=turn_taking_score,json=turnTakingScore,proto3" json:"turn_taking_score,omitempty"` // overall turn-taking quality + ResponseLatencyP50S float32 `protobuf:"fixed32,11,opt,name=response_latency_p50_s,json=responseLatencyP50S,proto3" json:"response_latency_p50_s,omitempty"` // reply latency from the audio; a gap is > 0, an overlap < 0 + ResponseLatencyP90S float32 `protobuf:"fixed32,12,opt,name=response_latency_p90_s,json=responseLatencyP90S,proto3" json:"response_latency_p90_s,omitempty"` + ResponseLatencyP95S float32 `protobuf:"fixed32,13,opt,name=response_latency_p95_s,json=responseLatencyP95S,proto3" json:"response_latency_p95_s,omitempty"` + AgentYieldLatencyS float32 `protobuf:"fixed32,14,opt,name=agent_yield_latency_s,json=agentYieldLatencyS,proto3" json:"agent_yield_latency_s,omitempty"` // seconds for the agent to stop after the caller barges in + EotMispredictionCount int32 `protobuf:"varint,15,opt,name=eot_misprediction_count,json=eotMispredictionCount,proto3" json:"eot_misprediction_count,omitempty"` // times the agent started before the caller's turn ended + OverlapRatio float32 `protobuf:"fixed32,16,opt,name=overlap_ratio,json=overlapRatio,proto3" json:"overlap_ratio,omitempty"` // overlapping speech / total speech + SilenceTotalS float32 `protobuf:"fixed32,17,opt,name=silence_total_s,json=silenceTotalS,proto3" json:"silence_total_s,omitempty"` // dead air within the conversation + AwkwardSilenceCount int32 `protobuf:"varint,18,opt,name=awkward_silence_count,json=awkwardSilenceCount,proto3" json:"awkward_silence_count,omitempty"` // inter-turn gaps beyond the natural-pause threshold + UnansweredCallerTurns int32 `protobuf:"varint,19,opt,name=unanswered_caller_turns,json=unansweredCallerTurns,proto3" json:"unanswered_caller_turns,omitempty"` // caller spoke but the agent never responded + FalseInterruptionCount int32 `protobuf:"varint,23,opt,name=false_interruption_count,json=falseInterruptionCount,proto3" json:"false_interruption_count,omitempty"` // agent paused for what was not a real interruption + FalseInterruptionUnrecoveredCount int32 `protobuf:"varint,24,opt,name=false_interruption_unrecovered_count,json=falseInterruptionUnrecoveredCount,proto3" json:"false_interruption_unrecovered_count,omitempty"` // of those, the ones it never resumed from + AgentE2ELatencyS float32 `protobuf:"fixed32,44,opt,name=agent_e2e_latency_s,json=agentE2eLatencyS,proto3" json:"agent_e2e_latency_s,omitempty"` // agent's self-reported end-to-end reply latency + InterruptionCount int32 `protobuf:"varint,51,opt,name=interruption_count,json=interruptionCount,proto3" json:"interruption_count,omitempty"` // total barge-ins, caller and agent + // -- simulator conduct -- + SimulatorEarlyTermination bool `protobuf:"varint,46,opt,name=simulator_early_termination,json=simulatorEarlyTermination,proto3" json:"simulator_early_termination,omitempty"` // the simulated caller ended a still-progressing conversation + SimulatorLateTermination bool `protobuf:"varint,47,opt,name=simulator_late_termination,json=simulatorLateTermination,proto3" json:"simulator_late_termination,omitempty"` // the simulated caller dragged on after the goal was met + // -- language quality -- + ConcisenessScore float32 `protobuf:"fixed32,20,opt,name=conciseness_score,json=concisenessScore,proto3" json:"conciseness_score,omitempty"` // brevity of the agent's replies + // -- run context -- + MeasuredTurnCount int32 `protobuf:"varint,50,opt,name=measured_turn_count,json=measuredTurnCount,proto3" json:"measured_turn_count,omitempty"` // turns carrying at least one measurement + HasRemoteSession bool `protobuf:"varint,52,opt,name=has_remote_session,json=hasRemoteSession,proto3" json:"has_remote_session,omitempty"` // false when only waveform-derived metrics are available + MetricsVersion string `protobuf:"bytes,53,opt,name=metrics_version,json=metricsVersion,proto3" json:"metrics_version,omitempty"` // metric-suite version; scores compare only within a version + JudgeModel string `protobuf:"bytes,54,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` // model used for judge-scored metrics + Turns []*SimulationRun_Job_AudioMetrics_TurnMetrics `protobuf:"bytes,60,rep,name=turns,proto3" json:"turns,omitempty"` // per-turn breakdown + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_Job_AudioMetrics) Reset() { @@ -1215,37 +1253,44 @@ func (*SimulationRun_Job_AudioMetrics) Descriptor() ([]byte, []int) { return file_livekit_agent_simulation_proto_rawDescGZIP(), []int{1, 0, 1} } -func (x *SimulationRun_Job_AudioMetrics) GetE2ELatencyP50S() float32 { +func (x *SimulationRun_Job_AudioMetrics) GetAccuracyScore() float32 { if x != nil { - return x.E2ELatencyP50S + return x.AccuracyScore } return 0 } -func (x *SimulationRun_Job_AudioMetrics) GetE2ELatencyP95S() float32 { +func (x *SimulationRun_Job_AudioMetrics) GetExperienceScore() float32 { if x != nil { - return x.E2ELatencyP95S + return x.ExperienceScore } return 0 } -func (x *SimulationRun_Job_AudioMetrics) GetWer() float32 { +func (x *SimulationRun_Job_AudioMetrics) GetSttWer() float32 { if x != nil { - return x.Wer + return x.SttWer } return 0 } -func (x *SimulationRun_Job_AudioMetrics) GetLlmTtftS() float32 { +func (x *SimulationRun_Job_AudioMetrics) GetSttCer() float32 { if x != nil { - return x.LlmTtftS + return x.SttCer } return 0 } -func (x *SimulationRun_Job_AudioMetrics) GetSttDelayS() float32 { +func (x *SimulationRun_Job_AudioMetrics) GetSttKeytermAccuracy() float32 { if x != nil { - return x.SttDelayS + return x.SttKeytermAccuracy + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetSttTranscriptionLatencyS() float32 { + if x != nil { + return x.SttTranscriptionLatencyS } return 0 } @@ -1257,20 +1302,258 @@ func (x *SimulationRun_Job_AudioMetrics) GetTtsTtfbS() float32 { return 0 } -func (x *SimulationRun_Job_AudioMetrics) GetInterruptions() int32 { +func (x *SimulationRun_Job_AudioMetrics) GetTtsSpeechRateWpm() float32 { + if x != nil { + return x.TtsSpeechRateWpm + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsNaturalnessScore() float32 { + if x != nil { + return x.TtsNaturalnessScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsProsodyScore() float32 { + if x != nil { + return x.TtsProsodyScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsExpressivenessScore() float32 { + if x != nil { + return x.TtsExpressivenessScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsEnunciationScore() float32 { + if x != nil { + return x.TtsEnunciationScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsUptalkScore() float32 { + if x != nil { + return x.TtsUptalkScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsWer() float32 { + if x != nil { + return x.TtsWer + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsConversationalityScore() float32 { + if x != nil { + return x.TtsConversationalityScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsHallucinationRate() float32 { + if x != nil { + return x.TtsHallucinationRate + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsAudioQualityMos() float32 { + if x != nil { + return x.TtsAudioQualityMos + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsVoiceConsistencyScore() float32 { + if x != nil { + return x.TtsVoiceConsistencyScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTtsSpokenArtifactCount() int32 { + if x != nil { + return x.TtsSpokenArtifactCount + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetLlmTtftS() float32 { + if x != nil { + return x.LlmTtftS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetLlmTtfsS() float32 { + if x != nil { + return x.LlmTtfsS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetLlmThroughputTps() float32 { + if x != nil { + return x.LlmThroughputTps + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetTurnTakingScore() float32 { + if x != nil { + return x.TurnTakingScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetResponseLatencyP50S() float32 { + if x != nil { + return x.ResponseLatencyP50S + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetResponseLatencyP90S() float32 { + if x != nil { + return x.ResponseLatencyP90S + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetResponseLatencyP95S() float32 { + if x != nil { + return x.ResponseLatencyP95S + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetAgentYieldLatencyS() float32 { + if x != nil { + return x.AgentYieldLatencyS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetEotMispredictionCount() int32 { + if x != nil { + return x.EotMispredictionCount + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetOverlapRatio() float32 { + if x != nil { + return x.OverlapRatio + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetSilenceTotalS() float32 { + if x != nil { + return x.SilenceTotalS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetAwkwardSilenceCount() int32 { + if x != nil { + return x.AwkwardSilenceCount + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetUnansweredCallerTurns() int32 { if x != nil { - return x.Interruptions + return x.UnansweredCallerTurns } return 0 } -func (x *SimulationRun_Job_AudioMetrics) GetSampleCount() int32 { +func (x *SimulationRun_Job_AudioMetrics) GetFalseInterruptionCount() int32 { if x != nil { - return x.SampleCount + return x.FalseInterruptionCount } return 0 } +func (x *SimulationRun_Job_AudioMetrics) GetFalseInterruptionUnrecoveredCount() int32 { + if x != nil { + return x.FalseInterruptionUnrecoveredCount + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetAgentE2ELatencyS() float32 { + if x != nil { + return x.AgentE2ELatencyS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetInterruptionCount() int32 { + if x != nil { + return x.InterruptionCount + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetSimulatorEarlyTermination() bool { + if x != nil { + return x.SimulatorEarlyTermination + } + return false +} + +func (x *SimulationRun_Job_AudioMetrics) GetSimulatorLateTermination() bool { + if x != nil { + return x.SimulatorLateTermination + } + return false +} + +func (x *SimulationRun_Job_AudioMetrics) GetConcisenessScore() float32 { + if x != nil { + return x.ConcisenessScore + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetMeasuredTurnCount() int32 { + if x != nil { + return x.MeasuredTurnCount + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics) GetHasRemoteSession() bool { + if x != nil { + return x.HasRemoteSession + } + return false +} + +func (x *SimulationRun_Job_AudioMetrics) GetMetricsVersion() string { + if x != nil { + return x.MetricsVersion + } + return "" +} + +func (x *SimulationRun_Job_AudioMetrics) GetJudgeModel() string { + if x != nil { + return x.JudgeModel + } + return "" +} + func (x *SimulationRun_Job_AudioMetrics) GetTurns() []*SimulationRun_Job_AudioMetrics_TurnMetrics { if x != nil { return x.Turns @@ -1279,19 +1562,19 @@ func (x *SimulationRun_Job_AudioMetrics) GetTurns() []*SimulationRun_Job_AudioMe } type SimulationRun_Job_AudioMetrics_TurnMetrics struct { - state protoimpl.MessageState `protogen:"open.v1"` - Turn int32 `protobuf:"varint,1,opt,name=turn,proto3" json:"turn,omitempty"` // 1-based turn index - // Tier 0 — true end-to-end response latency off the waveform. - ResponseLatencyS float32 `protobuf:"fixed32,2,opt,name=response_latency_s,json=responseLatencyS,proto3" json:"response_latency_s,omitempty"` - // Tier 1 — the agent's own pipeline, broadcast over its RemoteSession; - // -1 when the backchannel is absent. - Wer float32 `protobuf:"fixed32,3,opt,name=wer,proto3" json:"wer,omitempty"` - SttDelayS float32 `protobuf:"fixed32,4,opt,name=stt_delay_s,json=sttDelayS,proto3" json:"stt_delay_s,omitempty"` - LlmTtftS float32 `protobuf:"fixed32,5,opt,name=llm_ttft_s,json=llmTtftS,proto3" json:"llm_ttft_s,omitempty"` - TtsTtfbS float32 `protobuf:"fixed32,6,opt,name=tts_ttfb_s,json=ttsTtfbS,proto3" json:"tts_ttfb_s,omitempty"` - E2ELatencyS float32 `protobuf:"fixed32,7,opt,name=e2e_latency_s,json=e2eLatencyS,proto3" json:"e2e_latency_s,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Turn int32 `protobuf:"varint,1,opt,name=turn,proto3" json:"turn,omitempty"` // 1-based turn index + ResponseLatencyS float32 `protobuf:"fixed32,2,opt,name=response_latency_s,json=responseLatencyS,proto3" json:"response_latency_s,omitempty"` // reply latency from the audio + AgentCutIn bool `protobuf:"varint,3,opt,name=agent_cut_in,json=agentCutIn,proto3" json:"agent_cut_in,omitempty"` // agent started before the caller finished this turn + SttWer float32 `protobuf:"fixed32,4,opt,name=stt_wer,json=sttWer,proto3" json:"stt_wer,omitempty"` + SttTranscriptionLatencyS float32 `protobuf:"fixed32,5,opt,name=stt_transcription_latency_s,json=sttTranscriptionLatencyS,proto3" json:"stt_transcription_latency_s,omitempty"` + LlmTtftS float32 `protobuf:"fixed32,6,opt,name=llm_ttft_s,json=llmTtftS,proto3" json:"llm_ttft_s,omitempty"` + LlmTtfsS float32 `protobuf:"fixed32,10,opt,name=llm_ttfs_s,json=llmTtfsS,proto3" json:"llm_ttfs_s,omitempty"` + TtsTtfbS float32 `protobuf:"fixed32,7,opt,name=tts_ttfb_s,json=ttsTtfbS,proto3" json:"tts_ttfb_s,omitempty"` + AgentE2ELatencyS float32 `protobuf:"fixed32,8,opt,name=agent_e2e_latency_s,json=agentE2eLatencyS,proto3" json:"agent_e2e_latency_s,omitempty"` + ConcisenessScore float32 `protobuf:"fixed32,9,opt,name=conciseness_score,json=concisenessScore,proto3" json:"conciseness_score,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) Reset() { @@ -1338,16 +1621,23 @@ func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetResponseLatencyS() float return 0 } -func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetWer() float32 { +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetAgentCutIn() bool { + if x != nil { + return x.AgentCutIn + } + return false +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetSttWer() float32 { if x != nil { - return x.Wer + return x.SttWer } return 0 } -func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetSttDelayS() float32 { +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetSttTranscriptionLatencyS() float32 { if x != nil { - return x.SttDelayS + return x.SttTranscriptionLatencyS } return 0 } @@ -1359,6 +1649,13 @@ func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetLlmTtftS() float32 { return 0 } +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetLlmTtfsS() float32 { + if x != nil { + return x.LlmTtfsS + } + return 0 +} + func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetTtsTtfbS() float32 { if x != nil { return x.TtsTtfbS @@ -1366,9 +1663,16 @@ func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetTtsTtfbS() float32 { return 0 } -func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetE2ELatencyS() float32 { +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetAgentE2ELatencyS() float32 { + if x != nil { + return x.AgentE2ELatencyS + } + return 0 +} + +func (x *SimulationRun_Job_AudioMetrics_TurnMetrics) GetConcisenessScore() float32 { if x != nil { - return x.E2ELatencyS + return x.ConcisenessScore } return 0 } @@ -2076,7 +2380,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\n" + "suggestion\x18\x02 \x01(\tR\n" + "suggestion\x12\x14\n" + - "\x05label\x18\x03 \x01(\tR\x05label\"\x9b\x1c\n" + + "\x05label\x18\x03 \x01(\tR\x05label\"\xc9+\n" + "\rSimulationRun\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1d\n" + "\n" + @@ -2099,8 +2403,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x0fnum_simulations\x18\x0f \x01(\x05R\x0enumSimulations\x122\n" + "\x05usage\x18\x10 \x01(\v2\x1c.livekit.SimulationRun.UsageR\x05usage\x12 \n" + "\vconcurrency\x18\x11 \x01(\x05R\vconcurrency\x12+\n" + - "\x04mode\x18\x12 \x01(\x0e2\x17.livekit.SimulationModeR\x04mode\x1a\xb4\n" + - "\n" + + "\x04mode\x18\x12 \x01(\x0e2\x17.livekit.SimulationModeR\x04mode\x1a\xe2\x19\n" + "\x03Job\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x129\n" + "\x06status\x18\x02 \x01(\x0e2!.livekit.SimulationRun.Job.StatusR\x06status\x12\"\n" + @@ -2119,29 +2422,73 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\raudio_metrics\x18\x0e \x01(\v2'.livekit.SimulationRun.Job.AudioMetricsR\faudioMetrics\x1a]\n" + "\x05Usage\x12(\n" + "\x10text_turns_count\x18\x01 \x01(\x05R\x0etextTurnsCount\x12*\n" + - "\x11audio_turns_count\x18\x02 \x01(\x05R\x0faudioTurnsCount\x1a\xca\x04\n" + - "\fAudioMetrics\x12)\n" + - "\x11e2e_latency_p50_s\x18\x01 \x01(\x02R\x0ee2eLatencyP50S\x12)\n" + - "\x11e2e_latency_p95_s\x18\x02 \x01(\x02R\x0ee2eLatencyP95S\x12\x10\n" + - "\x03wer\x18\x03 \x01(\x02R\x03wer\x12\x1c\n" + + "\x11audio_turns_count\x18\x02 \x01(\x05R\x0faudioTurnsCount\x1a\xf8\x13\n" + + "\fAudioMetrics\x12%\n" + + "\x0eaccuracy_score\x18\x01 \x01(\x02R\raccuracyScore\x12)\n" + + "\x10experience_score\x18\x02 \x01(\x02R\x0fexperienceScore\x12\x17\n" + + "\astt_wer\x18! \x01(\x02R\x06sttWer\x12\x17\n" + + "\astt_cer\x18\" \x01(\x02R\x06sttCer\x120\n" + + "\x14stt_keyterm_accuracy\x18# \x01(\x02R\x12sttKeytermAccuracy\x12=\n" + + "\x1bstt_transcription_latency_s\x18( \x01(\x02R\x18sttTranscriptionLatencyS\x12\x1c\n" + + "\n" + + "tts_ttfb_s\x18+ \x01(\x02R\bttsTtfbS\x12-\n" + + "\x13tts_speech_rate_wpm\x18- \x01(\x02R\x10ttsSpeechRateWpm\x122\n" + + "\x15tts_naturalness_score\x18\x16 \x01(\x02R\x13ttsNaturalnessScore\x12*\n" + + "\x11tts_prosody_score\x18% \x01(\x02R\x0fttsProsodyScore\x128\n" + + "\x18tts_expressiveness_score\x18& \x01(\x02R\x16ttsExpressivenessScore\x122\n" + + "\x15tts_enunciation_score\x18 \x01(\x02R\x13ttsEnunciationScore\x12(\n" + + "\x10tts_uptalk_score\x18' \x01(\x02R\x0ettsUptalkScore\x12\x17\n" + + "\atts_wer\x18$ \x01(\x02R\x06ttsWer\x12>\n" + + "\x1btts_conversationality_score\x187 \x01(\x02R\x19ttsConversationalityScore\x124\n" + + "\x16tts_hallucination_rate\x188 \x01(\x02R\x14ttsHallucinationRate\x121\n" + + "\x15tts_audio_quality_mos\x189 \x01(\x02R\x12ttsAudioQualityMos\x12=\n" + + "\x1btts_voice_consistency_score\x18: \x01(\x02R\x18ttsVoiceConsistencyScore\x129\n" + + "\x19tts_spoken_artifact_count\x18; \x01(\x05R\x16ttsSpokenArtifactCount\x12\x1c\n" + "\n" + - "llm_ttft_s\x18\x04 \x01(\x02R\bllmTtftS\x12\x1e\n" + - "\vstt_delay_s\x18\x05 \x01(\x02R\tsttDelayS\x12\x1c\n" + + "llm_ttft_s\x18) \x01(\x02R\bllmTtftS\x12\x1c\n" + "\n" + - "tts_ttfb_s\x18\x06 \x01(\x02R\bttsTtfbS\x12$\n" + - "\rinterruptions\x18\a \x01(\x05R\rinterruptions\x12!\n" + - "\fsample_count\x18\b \x01(\x05R\vsampleCount\x12I\n" + - "\x05turns\x18\t \x03(\v23.livekit.SimulationRun.Job.AudioMetrics.TurnMetricsR\x05turns\x1a\xe1\x01\n" + + "llm_ttfs_s\x180 \x01(\x02R\bllmTtfsS\x12,\n" + + "\x12llm_throughput_tps\x18* \x01(\x02R\x10llmThroughputTps\x12*\n" + + "\x11turn_taking_score\x18\n" + + " \x01(\x02R\x0fturnTakingScore\x123\n" + + "\x16response_latency_p50_s\x18\v \x01(\x02R\x13responseLatencyP50S\x123\n" + + "\x16response_latency_p90_s\x18\f \x01(\x02R\x13responseLatencyP90S\x123\n" + + "\x16response_latency_p95_s\x18\r \x01(\x02R\x13responseLatencyP95S\x121\n" + + "\x15agent_yield_latency_s\x18\x0e \x01(\x02R\x12agentYieldLatencyS\x126\n" + + "\x17eot_misprediction_count\x18\x0f \x01(\x05R\x15eotMispredictionCount\x12#\n" + + "\roverlap_ratio\x18\x10 \x01(\x02R\foverlapRatio\x12&\n" + + "\x0fsilence_total_s\x18\x11 \x01(\x02R\rsilenceTotalS\x122\n" + + "\x15awkward_silence_count\x18\x12 \x01(\x05R\x13awkwardSilenceCount\x126\n" + + "\x17unanswered_caller_turns\x18\x13 \x01(\x05R\x15unansweredCallerTurns\x128\n" + + "\x18false_interruption_count\x18\x17 \x01(\x05R\x16falseInterruptionCount\x12O\n" + + "$false_interruption_unrecovered_count\x18\x18 \x01(\x05R!falseInterruptionUnrecoveredCount\x12-\n" + + "\x13agent_e2e_latency_s\x18, \x01(\x02R\x10agentE2eLatencyS\x12-\n" + + "\x12interruption_count\x183 \x01(\x05R\x11interruptionCount\x12>\n" + + "\x1bsimulator_early_termination\x18. \x01(\bR\x19simulatorEarlyTermination\x12<\n" + + "\x1asimulator_late_termination\x18/ \x01(\bR\x18simulatorLateTermination\x12+\n" + + "\x11conciseness_score\x18\x14 \x01(\x02R\x10concisenessScore\x12.\n" + + "\x13measured_turn_count\x182 \x01(\x05R\x11measuredTurnCount\x12,\n" + + "\x12has_remote_session\x184 \x01(\bR\x10hasRemoteSession\x12'\n" + + "\x0fmetrics_version\x185 \x01(\tR\x0emetricsVersion\x12\x1f\n" + + "\vjudge_model\x186 \x01(\tR\n" + + "judgeModel\x12I\n" + + "\x05turns\x18< \x03(\v23.livekit.SimulationRun.Job.AudioMetrics.TurnMetricsR\x05turns\x1a\xff\x02\n" + "\vTurnMetrics\x12\x12\n" + "\x04turn\x18\x01 \x01(\x05R\x04turn\x12,\n" + - "\x12response_latency_s\x18\x02 \x01(\x02R\x10responseLatencyS\x12\x10\n" + - "\x03wer\x18\x03 \x01(\x02R\x03wer\x12\x1e\n" + - "\vstt_delay_s\x18\x04 \x01(\x02R\tsttDelayS\x12\x1c\n" + + "\x12response_latency_s\x18\x02 \x01(\x02R\x10responseLatencyS\x12 \n" + + "\fagent_cut_in\x18\x03 \x01(\bR\n" + + "agentCutIn\x12\x17\n" + + "\astt_wer\x18\x04 \x01(\x02R\x06sttWer\x12=\n" + + "\x1bstt_transcription_latency_s\x18\x05 \x01(\x02R\x18sttTranscriptionLatencyS\x12\x1c\n" + + "\n" + + "llm_ttft_s\x18\x06 \x01(\x02R\bllmTtftS\x12\x1c\n" + "\n" + - "llm_ttft_s\x18\x05 \x01(\x02R\bllmTtftS\x12\x1c\n" + + "llm_ttfs_s\x18\n" + + " \x01(\x02R\bllmTtfsS\x12\x1c\n" + "\n" + - "tts_ttfb_s\x18\x06 \x01(\x02R\bttsTtfbS\x12\"\n" + - "\re2e_latency_s\x18\a \x01(\x02R\ve2eLatencyS\"o\n" + + "tts_ttfb_s\x18\a \x01(\x02R\bttsTtfbS\x12-\n" + + "\x13agent_e2e_latency_s\x18\b \x01(\x02R\x10agentE2eLatencyS\x12+\n" + + "\x11conciseness_score\x18\t \x01(\x02R\x10concisenessScoreJ\x04\b\x15\x10\x16J\x04\b\x1e\x10\x1fJ\x04\b\x1f\x10 \"o\n" + "\x06Status\x12\x12\n" + "\x0eSTATUS_PENDING\x10\x00\x12\x12\n" + "\x0eSTATUS_RUNNING\x10\x01\x12\x14\n" + diff --git a/protobufs/livekit_agent_simulation.proto b/protobufs/livekit_agent_simulation.proto index 8fbd1a939..c7f26010c 100644 --- a/protobufs/livekit_agent_simulation.proto +++ b/protobufs/livekit_agent_simulation.proto @@ -72,9 +72,8 @@ message SimulationRun { google.protobuf.Timestamp ended_at = 11; string room_id = 12; Usage usage = 13; - // Audio-mode pipeline metrics aggregated over the call's measured turns. - // Only populated for audio jobs. A -1 field means the datum was - // unavailable (Tier-1 metrics require the agent's backchannel). + // Metrics for an audio job, aggregated over the call's measured turns. + // Unset for text jobs; a -1 field was not measured (see AudioMetrics). AudioMetrics audio_metrics = 14; message Usage { @@ -82,35 +81,83 @@ message SimulationRun { int32 audio_turns_count = 2; } + // Per-call audio metrics, grouped by pipeline stage. A -1 means the metric + // was not measured for this call; for counts, Scores are 0-1 unless noted. message AudioMetrics { - // Tier 0 — true end-to-end response latency off the waveform (our - // playout end -> the agent's speech onset). Always present for audio. - float e2e_latency_p50_s = 1; - float e2e_latency_p95_s = 2; - // Tier 1 — mean over turns; -1 when the backchannel is absent. - float wer = 3; // normalized word error rate of the agent's STT of us - float llm_ttft_s = 4; - float stt_delay_s = 5; - float tts_ttfb_s = 6; - // Call-level barge-in count. - int32 interruptions = 7; - // Turns that contributed measurements (context for the aggregates). - int32 sample_count = 8; - // Per-turn breakdown behind the aggregates above, one entry per measured - // turn in order. A -1 field means the datum was unavailable for that turn. - repeated TurnMetrics turns = 9; + reserved 21, 30, 31; + + // -- overall -- + float accuracy_score = 1; // how accurately the agent achieved the intended outcome + float experience_score = 2; // overall quality of the caller experience + + // -- speech recognition -- + float stt_wer = 33; // word error rate of the agent's transcription of the caller + float stt_cer = 34; // character error rate + float stt_keyterm_accuracy = 35; // recall of key terms (names, IDs, amounts) in the transcription + float stt_transcription_latency_s = 40; // seconds from the end of caller speech to the final transcript + + // -- speech synthesis -- + float tts_ttfb_s = 43; // seconds to the first audio of a reply + float tts_speech_rate_wpm = 45; // speaking rate, words per minute + float tts_naturalness_score = 22; // overall perceptual naturalness + float tts_prosody_score = 37; // pitch and intonation + float tts_expressiveness_score = 38; // expressiveness and affect + float tts_enunciation_score = 32; // clarity of structured entities (emails, phone numbers, codes) + float tts_uptalk_score = 39; // appropriate terminal intonation; low = inappropriate uptalk + float tts_wer = 36; // intelligibility: intended text vs a transcription of the output audio + float tts_conversationality_score = 55; // sounds conversational rather than read-aloud + float tts_hallucination_rate = 56; // fraction of output audio not traceable to the input text + float tts_audio_quality_mos = 57; // 1-5 mean opinion score (coloration, noise, discontinuity, loudness) + float tts_voice_consistency_score = 58; // voice stays consistent within and across turns + int32 tts_spoken_artifact_count = 59; // spoken markup, clicks, or vowel prolongation + + // -- language model -- + float llm_ttft_s = 41; // seconds to the first token + float llm_ttfs_s = 48; // seconds to the first complete sentence + float llm_throughput_tps = 42; // output tokens per second + + // -- conversation timing & turn-taking -- + float turn_taking_score = 10; // overall turn-taking quality + float response_latency_p50_s = 11; // reply latency from the audio; a gap is > 0, an overlap < 0 + float response_latency_p90_s = 12; + float response_latency_p95_s = 13; + float agent_yield_latency_s = 14; // seconds for the agent to stop after the caller barges in + int32 eot_misprediction_count = 15; // times the agent started before the caller's turn ended + float overlap_ratio = 16; // overlapping speech / total speech + float silence_total_s = 17; // dead air within the conversation + int32 awkward_silence_count = 18; // inter-turn gaps beyond the natural-pause threshold + int32 unanswered_caller_turns = 19; // caller spoke but the agent never responded + int32 false_interruption_count = 23; // agent paused for what was not a real interruption + int32 false_interruption_unrecovered_count = 24; // of those, the ones it never resumed from + float agent_e2e_latency_s = 44; // agent's self-reported end-to-end reply latency + int32 interruption_count = 51; // total barge-ins, caller and agent + + // -- simulator conduct -- + bool simulator_early_termination = 46; // the simulated caller ended a still-progressing conversation + bool simulator_late_termination = 47; // the simulated caller dragged on after the goal was met + + // -- language quality -- + float conciseness_score = 20; // brevity of the agent's replies + + // -- run context -- + int32 measured_turn_count = 50; // turns carrying at least one measurement + bool has_remote_session = 52; // false when only waveform-derived metrics are available + string metrics_version = 53; // metric-suite version; scores compare only within a version + string judge_model = 54; // model used for judge-scored metrics + + repeated TurnMetrics turns = 60; // per-turn breakdown message TurnMetrics { - int32 turn = 1; // 1-based turn index - // Tier 0 — true end-to-end response latency off the waveform. - float response_latency_s = 2; - // Tier 1 — the agent's own pipeline, broadcast over its RemoteSession; - // -1 when the backchannel is absent. - float wer = 3; - float stt_delay_s = 4; - float llm_ttft_s = 5; - float tts_ttfb_s = 6; - float e2e_latency_s = 7; + int32 turn = 1; // 1-based turn index + float response_latency_s = 2; // reply latency from the audio + bool agent_cut_in = 3; // agent started before the caller finished this turn + float stt_wer = 4; + float stt_transcription_latency_s = 5; + float llm_ttft_s = 6; + float llm_ttfs_s = 10; + float tts_ttfb_s = 7; + float agent_e2e_latency_s = 8; + float conciseness_score = 9; } } } From 715b9162958a0f6bab10d9bae1e3b6e447d84e5b Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 20:10:43 +0000 Subject: [PATCH 5/5] generated protobuf --- livekit/livekit_agent_simulation.pb.go | 5 +- livekit/livekit_agent_simulation.twirp.go | 308 +++++++++++++--------- 2 files changed, 184 insertions(+), 129 deletions(-) diff --git a/livekit/livekit_agent_simulation.pb.go b/livekit/livekit_agent_simulation.pb.go index 14169f16e..a7cab10f1 100644 --- a/livekit/livekit_agent_simulation.pb.go +++ b/livekit/livekit_agent_simulation.pb.go @@ -15,7 +15,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v7.35.0 +// protoc v7.35.1 // source: livekit_agent_simulation.proto package livekit @@ -1163,8 +1163,7 @@ func (x *SimulationRun_Job_Usage) GetAudioTurnsCount() int32 { } // Per-call audio metrics, grouped by pipeline stage. A -1 means the metric -// was not measured for this call; for counts, -1 = not measured and 0 = a -// measured zero. Scores are 0-1 unless noted. +// was not measured for this call; for counts, Scores are 0-1 unless noted. type SimulationRun_Job_AudioMetrics struct { state protoimpl.MessageState `protogen:"open.v1"` // -- overall -- diff --git a/livekit/livekit_agent_simulation.twirp.go b/livekit/livekit_agent_simulation.twirp.go index 0b1770863..f59ef26e1 100644 --- a/livekit/livekit_agent_simulation.twirp.go +++ b/livekit/livekit_agent_simulation.twirp.go @@ -1912,130 +1912,186 @@ func (s *agentSimulationServer) PathPrefix() string { } var twirpFileDescriptor9 = []byte{ - // 1993 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x6f, 0xdb, 0xc8, - 0x15, 0x0f, 0xa9, 0x0f, 0x4b, 0x4f, 0x96, 0x2c, 0xcf, 0x3a, 0x59, 0x85, 0xd9, 0x64, 0x1d, 0x67, - 0xb7, 0x71, 0x37, 0x5d, 0x39, 0xd0, 0xd6, 0xdd, 0x3a, 0x6d, 0x81, 0xca, 0xb6, 0x62, 0x2b, 0xf5, - 0x57, 0x29, 0x19, 0x6d, 0x53, 0x14, 0x04, 0x45, 0x8e, 0x65, 0x3a, 0x14, 0x47, 0xe5, 0x0c, 0xbd, - 0xf1, 0xa1, 0xc7, 0x02, 0x01, 0x7a, 0xea, 0xb9, 0x40, 0xd1, 0xcb, 0x1e, 0xda, 0x6b, 0xaf, 0x05, - 0xfa, 0x37, 0xf4, 0xde, 0xbf, 0xa0, 0xe8, 0xb5, 0xf7, 0x62, 0x3e, 0x28, 0x91, 0x92, 0x2c, 0x7b, - 0x17, 0xc8, 0x4d, 0xf3, 0x9b, 0xdf, 0xbc, 0x79, 0xf3, 0xe6, 0xf7, 0xde, 0x1b, 0x0a, 0x1e, 0xf9, - 0xde, 0x25, 0x7e, 0xe3, 0x31, 0xcb, 0xee, 0xe3, 0x80, 0x59, 0xd4, 0x1b, 0x44, 0xbe, 0xcd, 0x3c, - 0x12, 0xd4, 0x87, 0x21, 0x61, 0x04, 0x2d, 0xa8, 0x79, 0xe3, 0xb1, 0x20, 0x6c, 0x4c, 0xd0, 0x31, - 0xa5, 0x23, 0xae, 0xf1, 0x71, 0x9f, 0x90, 0xbe, 0x8f, 0x37, 0xc4, 0xa8, 0x17, 0x9d, 0x6d, 0x30, - 0x6f, 0x80, 0x29, 0xb3, 0x07, 0x43, 0x45, 0xb8, 0x1f, 0xaf, 0x76, 0x7c, 0x12, 0xb9, 0xd2, 0x86, - 0x9a, 0x5a, 0x89, 0xa7, 0x06, 0xc4, 0xc5, 0x3e, 0x95, 0xe8, 0xda, 0xbf, 0x32, 0xb0, 0xd2, 0x19, - 0xb9, 0x64, 0x46, 0x41, 0x27, 0x1a, 0x0c, 0xec, 0xf0, 0x0a, 0xdd, 0x83, 0xfc, 0xd0, 0xa6, 0x14, - 0xbb, 0x35, 0x6d, 0x55, 0x5b, 0xcf, 0x99, 0x6a, 0xc4, 0xf1, 0x33, 0xdb, 0xf3, 0xb1, 0x5b, 0xd3, - 0x25, 0x2e, 0x47, 0xe8, 0x21, 0x40, 0x9f, 0x78, 0x41, 0xdf, 0xfa, 0x0a, 0xfb, 0x7e, 0x2d, 0xb3, - 0xaa, 0xad, 0x17, 0xcd, 0xa2, 0x40, 0x7e, 0x81, 0x7d, 0x9f, 0x4f, 0x33, 0x62, 0x79, 0x83, 0x61, - 0x48, 0x2e, 0x71, 0x2d, 0x2b, 0xa7, 0x19, 0x69, 0x4b, 0x00, 0xfd, 0x08, 0xf2, 0x1e, 0xa5, 0x11, - 0xa6, 0xb5, 0xdc, 0x6a, 0x66, 0xbd, 0xd4, 0x78, 0x52, 0x57, 0xde, 0xd6, 0x67, 0x39, 0x57, 0x6f, - 0x73, 0xae, 0xa9, 0x96, 0xa0, 0x9f, 0xc3, 0xa2, 0x73, 0x6e, 0x33, 0xeb, 0xdc, 0xa3, 0x8c, 0x84, - 0x57, 0xb5, 0xbc, 0x30, 0x51, 0x9f, 0x6f, 0x62, 0xe7, 0xdc, 0x66, 0xfb, 0x72, 0x41, 0x2b, 0x60, - 0xe1, 0x95, 0x59, 0x72, 0xc6, 0x88, 0xf1, 0x1a, 0xaa, 0x93, 0x04, 0x54, 0x85, 0xcc, 0x1b, 0x7c, - 0x25, 0xc2, 0x51, 0x34, 0xf9, 0x4f, 0xf4, 0x1c, 0x72, 0x97, 0xb6, 0x1f, 0x61, 0x11, 0x8a, 0x52, - 0xc3, 0x18, 0xed, 0x28, 0xe3, 0xce, 0x2d, 0xec, 0x90, 0x80, 0xe1, 0xb7, 0xcc, 0x94, 0xc4, 0x17, - 0xfa, 0x0f, 0x35, 0xc3, 0x82, 0x9c, 0xf0, 0x1f, 0xad, 0x42, 0xc9, 0xc5, 0xd4, 0x09, 0xbd, 0x21, - 0xf7, 0x4d, 0x19, 0x4e, 0x42, 0xe8, 0x11, 0x00, 0x8d, 0xfa, 0x7d, 0x4c, 0x05, 0x41, 0x17, 0x84, - 0x04, 0x82, 0x56, 0x20, 0xe7, 0xdb, 0x3d, 0x1c, 0xc7, 0x5b, 0x0e, 0xd6, 0xfe, 0x7b, 0x1f, 0xca, - 0xa9, 0x33, 0xa3, 0x0a, 0xe8, 0x9e, 0xab, 0x36, 0xd0, 0x3d, 0x71, 0x59, 0xc3, 0x90, 0x5c, 0x60, - 0x87, 0x59, 0x9e, 0xab, 0xec, 0x16, 0x15, 0xd2, 0x76, 0xd1, 0x26, 0xe4, 0x29, 0xb3, 0x59, 0x44, - 0x85, 0xdd, 0x4a, 0xe3, 0xe1, 0xec, 0x50, 0xd6, 0x3b, 0x82, 0x64, 0x2a, 0x32, 0x7a, 0x06, 0xcb, - 0x52, 0xb4, 0xc9, 0x53, 0xc9, 0xab, 0xae, 0x8a, 0x89, 0xdd, 0xc4, 0xd1, 0x56, 0x20, 0x87, 0xc3, - 0x90, 0x84, 0xb5, 0x9c, 0x74, 0x5d, 0x0c, 0xd0, 0x16, 0x80, 0x13, 0x62, 0x9b, 0x61, 0xd7, 0xb2, - 0x59, 0x2d, 0xaf, 0xc2, 0x2a, 0x55, 0x5f, 0x8f, 0x55, 0x5f, 0xef, 0xc6, 0xaa, 0x37, 0x8b, 0x8a, - 0xdd, 0x64, 0xa8, 0x0e, 0xd9, 0x0b, 0xd2, 0xa3, 0xb5, 0x05, 0x71, 0xfb, 0xc6, 0x35, 0x2e, 0xbf, - 0x22, 0x3d, 0x53, 0xf0, 0xd0, 0x97, 0xb0, 0x40, 0xa5, 0x16, 0x6a, 0x05, 0xb1, 0xcf, 0xc3, 0xb9, - 0x82, 0x31, 0x63, 0x36, 0x0f, 0x9e, 0x3c, 0x66, 0x60, 0x0f, 0x70, 0xad, 0x28, 0x83, 0x27, 0x90, - 0x23, 0x7b, 0x80, 0xd1, 0x4f, 0xa0, 0x42, 0x1d, 0x1c, 0xd8, 0xa1, 0x47, 0xac, 0x7e, 0x48, 0xa2, - 0x61, 0x0d, 0x84, 0xf9, 0x7b, 0x63, 0xf3, 0x6a, 0x7a, 0x8f, 0xcf, 0x9a, 0x65, 0x9a, 0x1c, 0xa2, - 0x4d, 0x28, 0xe0, 0xc0, 0x95, 0xe7, 0x2f, 0xdd, 0x78, 0xfe, 0x05, 0xc1, 0x6d, 0x32, 0xf4, 0x00, - 0x8a, 0x17, 0xa4, 0x67, 0x39, 0x24, 0x0a, 0x58, 0x6d, 0x51, 0x64, 0x66, 0xe1, 0x82, 0xf4, 0x76, - 0xf8, 0x18, 0x3d, 0x86, 0x45, 0x99, 0xbd, 0x6a, 0xbe, 0x2c, 0xe6, 0x4b, 0x12, 0x1b, 0x51, 0x64, - 0x22, 0x2b, 0x4a, 0x45, 0x52, 0x24, 0x26, 0x29, 0x4f, 0x61, 0x29, 0x88, 0x06, 0x89, 0x02, 0x46, - 0x6b, 0x4b, 0x82, 0x55, 0x09, 0xa2, 0xc1, 0x38, 0x64, 0x14, 0x35, 0x20, 0x17, 0x51, 0xbb, 0x8f, - 0x6b, 0x55, 0xe1, 0xff, 0x47, 0xd7, 0x5c, 0xc5, 0x29, 0xe7, 0x98, 0x92, 0xca, 0x73, 0xc1, 0x21, - 0x81, 0x13, 0x85, 0x21, 0x0e, 0x9c, 0xab, 0xda, 0xb2, 0xdc, 0x3e, 0x01, 0xa1, 0x67, 0x90, 0xe5, - 0x95, 0xab, 0x86, 0x84, 0x24, 0x3f, 0x9c, 0x61, 0xf4, 0x90, 0xb8, 0xd8, 0x14, 0x24, 0xe3, 0xf7, - 0x05, 0xc8, 0xbc, 0x22, 0xbd, 0x29, 0xe1, 0x6f, 0x8d, 0x94, 0xad, 0x0b, 0x33, 0x8f, 0xaf, 0x97, - 0xc9, 0xa4, 0xba, 0xd7, 0x60, 0xd1, 0x0b, 0x28, 0x0b, 0x23, 0x47, 0x9e, 0x5d, 0xa6, 0x5c, 0x0a, - 0x1b, 0x8b, 0x3a, 0x9b, 0x14, 0xf5, 0xe7, 0x80, 0xa4, 0x60, 0xf0, 0xdb, 0x21, 0x76, 0x98, 0x8a, - 0x9d, 0xd4, 0xbd, 0xcc, 0x98, 0x56, 0x62, 0x62, 0x9c, 0xd4, 0xf9, 0x44, 0x52, 0x23, 0x04, 0x59, - 0x66, 0xf7, 0xa5, 0xbc, 0x8b, 0xa6, 0xf8, 0xcd, 0x2f, 0x3d, 0x24, 0x64, 0x20, 0x85, 0x58, 0x10, - 0xec, 0x02, 0x07, 0x84, 0x0e, 0xb7, 0x00, 0x28, 0xb3, 0x43, 0x95, 0x4a, 0x70, 0x73, 0x2a, 0x29, - 0x76, 0x93, 0x7d, 0x5b, 0x0d, 0x7e, 0x08, 0x0b, 0xc2, 0x1d, 0xcf, 0x15, 0x0a, 0x2c, 0x9a, 0x79, - 0x3e, 0x6c, 0xbb, 0xe8, 0x07, 0xb1, 0x20, 0xca, 0xc2, 0xd8, 0xea, 0x9c, 0xa0, 0xa7, 0x44, 0x71, - 0x00, 0x65, 0x3b, 0x72, 0x3d, 0x62, 0x0d, 0x30, 0x0b, 0x3d, 0x87, 0x0a, 0x55, 0x96, 0x1a, 0x4f, - 0xe7, 0xac, 0x6f, 0x72, 0xfe, 0xa1, 0xa4, 0x9b, 0x8b, 0x76, 0x62, 0x64, 0xfc, 0x06, 0x72, 0xc2, - 0x3a, 0x5a, 0x87, 0x2a, 0xaf, 0xc9, 0x16, 0x8b, 0xc2, 0x80, 0x2a, 0xbd, 0xcb, 0x26, 0x57, 0xe1, - 0x78, 0x97, 0xc3, 0x52, 0xf2, 0x9f, 0xc1, 0xb2, 0x74, 0x20, 0x49, 0x95, 0x7d, 0x6f, 0x49, 0x4c, - 0x8c, 0xb9, 0xc6, 0x9f, 0x74, 0x58, 0x4c, 0xee, 0x8e, 0xbe, 0x0b, 0xcb, 0xb8, 0x81, 0x2d, 0xdf, - 0x66, 0x5c, 0xbf, 0xd6, 0x70, 0xf3, 0xb9, 0x45, 0xc5, 0x3e, 0xba, 0x59, 0xc1, 0x0d, 0x7c, 0x20, - 0xf1, 0x93, 0xcd, 0xe7, 0x9d, 0x29, 0xea, 0xd6, 0xa6, 0x25, 0x15, 0x9a, 0xa6, 0x6e, 0x6d, 0x76, - 0x78, 0x17, 0xfa, 0x0a, 0x87, 0x42, 0x7d, 0xba, 0xc9, 0x7f, 0xa2, 0x8f, 0x00, 0x7c, 0x7f, 0x60, - 0x31, 0x76, 0xc6, 0x2c, 0x2a, 0x94, 0xa7, 0x9b, 0x05, 0xdf, 0x1f, 0x74, 0xd9, 0x19, 0xeb, 0xa0, - 0x47, 0x50, 0xa2, 0x8c, 0x97, 0x64, 0xdf, 0xbe, 0xb2, 0xa4, 0xea, 0x74, 0x7e, 0xd7, 0x6c, 0x97, - 0x23, 0x1d, 0xbe, 0x9a, 0x31, 0xca, 0x57, 0xf7, 0x2c, 0x2a, 0x24, 0xa7, 0x9b, 0x05, 0xc6, 0x68, - 0x97, 0x9d, 0xf5, 0x3a, 0xe8, 0x13, 0x28, 0x7b, 0x01, 0xc3, 0x61, 0x18, 0x0d, 0xa5, 0x6a, 0x17, - 0xc4, 0xe1, 0xd3, 0x20, 0x2f, 0x1e, 0xd4, 0x1e, 0x0c, 0x7d, 0xac, 0x22, 0x54, 0x90, 0xd9, 0x2b, - 0x31, 0x11, 0x9d, 0x35, 0x02, 0x79, 0x99, 0x4f, 0x08, 0x41, 0xa5, 0xd3, 0x6d, 0x76, 0x4f, 0x3b, - 0xd6, 0x49, 0xeb, 0x68, 0xb7, 0x7d, 0xb4, 0x57, 0xbd, 0x93, 0xc0, 0xcc, 0xd3, 0xa3, 0x23, 0x8e, - 0x69, 0x68, 0x05, 0xaa, 0x0a, 0xdb, 0x39, 0x3e, 0x3c, 0x39, 0x68, 0x75, 0x5b, 0xbb, 0x55, 0x1d, - 0x2d, 0x43, 0x59, 0xa1, 0x2f, 0x9b, 0xed, 0x83, 0xd6, 0x6e, 0x35, 0x93, 0x24, 0x36, 0x8f, 0x76, - 0x5a, 0x07, 0x1c, 0xcd, 0xbe, 0xca, 0x16, 0x8a, 0x55, 0x30, 0xfe, 0x97, 0x81, 0xfc, 0x8e, 0x68, - 0x11, 0xc6, 0xbf, 0x75, 0x58, 0x30, 0xf1, 0x6f, 0x23, 0x4c, 0xd9, 0x44, 0xff, 0xd3, 0x26, 0xfb, - 0x5f, 0xba, 0xc2, 0xeb, 0x93, 0x15, 0x7e, 0x46, 0x21, 0xcc, 0xce, 0x2c, 0x84, 0xf7, 0x20, 0x1f, - 0xe2, 0x3e, 0xef, 0x82, 0x79, 0x95, 0x0f, 0x62, 0x84, 0xb6, 0xa7, 0x5a, 0xc4, 0xc2, 0xbc, 0x16, - 0xb1, 0x7f, 0x67, 0xa2, 0x49, 0xbc, 0xd3, 0x34, 0xf4, 0x69, 0xba, 0x60, 0x8a, 0x90, 0xef, 0x6b, - 0xa9, 0x92, 0xc9, 0x69, 0x71, 0xd5, 0x2c, 0xde, 0xa2, 0x6a, 0x6e, 0x2f, 0xc3, 0x92, 0x95, 0x76, - 0x6c, 0xbb, 0x02, 0x8b, 0x56, 0xc2, 0xe4, 0xab, 0x6c, 0x21, 0x53, 0xcd, 0x9a, 0xd3, 0x7d, 0xde, - 0xf8, 0x83, 0x06, 0x05, 0x13, 0xd3, 0x21, 0x09, 0x28, 0xe6, 0x79, 0x33, 0x8e, 0x8e, 0x15, 0x46, - 0xc1, 0x38, 0xcc, 0x4b, 0x34, 0x99, 0xb3, 0x6d, 0x17, 0x75, 0xe0, 0xde, 0x30, 0xc4, 0xd4, 0xeb, - 0x07, 0xd8, 0xb5, 0x86, 0x84, 0x32, 0x2b, 0x94, 0xb7, 0xa4, 0x5e, 0x55, 0xe3, 0xb6, 0x7c, 0x12, - 0xd3, 0x4e, 0x08, 0x65, 0xea, 0x2a, 0xcd, 0x95, 0xe1, 0x0c, 0xd4, 0xf8, 0x8b, 0x06, 0x1f, 0xec, - 0x90, 0xe0, 0xcc, 0x0b, 0x07, 0x1d, 0x12, 0x85, 0x0e, 0x3e, 0x1d, 0xfa, 0xc4, 0x76, 0x8d, 0xdf, - 0xdd, 0x5a, 0x03, 0x33, 0x8f, 0xa0, 0xcf, 0x3e, 0xc2, 0x53, 0x58, 0x72, 0x88, 0x8b, 0x2d, 0xcc, - 0xdf, 0x89, 0x43, 0xe2, 0x05, 0x4c, 0x75, 0x87, 0x0a, 0x87, 0x5b, 0x23, 0xd4, 0x80, 0x71, 0x8c, - 0x8c, 0x3f, 0x6a, 0x90, 0xd9, 0xc3, 0xcc, 0xe8, 0xbe, 0x0f, 0x97, 0x8c, 0xef, 0x27, 0x6e, 0x63, - 0x1d, 0x32, 0x61, 0x24, 0xdf, 0x97, 0x29, 0x8d, 0x25, 0x97, 0x98, 0x9c, 0x62, 0xfc, 0x53, 0x87, - 0xec, 0x81, 0x47, 0x99, 0xf1, 0x0f, 0xed, 0xd6, 0x5e, 0xbd, 0x98, 0x68, 0xa9, 0xf3, 0x1f, 0x8b, - 0xfb, 0x77, 0xe2, 0x86, 0xca, 0xd5, 0xf9, 0x63, 0x80, 0xa1, 0xdd, 0xc7, 0x16, 0x23, 0x6f, 0x70, - 0x20, 0x62, 0x56, 0x6a, 0xd4, 0x46, 0xeb, 0xbb, 0x1c, 0x3d, 0xb1, 0xfb, 0x5e, 0x20, 0x8c, 0xec, - 0x6b, 0x66, 0x91, 0xb3, 0x05, 0xfc, 0x4e, 0xd3, 0xb6, 0x8b, 0xb0, 0x60, 0x49, 0x5b, 0xdb, 0x65, - 0x28, 0x59, 0x63, 0x4b, 0xc6, 0xdb, 0x94, 0x16, 0xb3, 0x61, 0x14, 0xf0, 0xca, 0x9b, 0x99, 0x73, - 0x7c, 0xc1, 0x41, 0x3f, 0x85, 0xa5, 0x80, 0x77, 0x86, 0x84, 0x53, 0xfa, 0x7c, 0xa7, 0xcc, 0x32, - 0x5f, 0x70, 0x12, 0xbb, 0x65, 0x5c, 0x40, 0x7e, 0xc7, 0x0e, 0x1c, 0xec, 0xbf, 0xa7, 0x7b, 0x4d, - 0x2a, 0xe8, 0xfd, 0x36, 0xb4, 0xb5, 0xaf, 0xb5, 0x51, 0xcd, 0xbe, 0x0f, 0x77, 0xd3, 0x35, 0xdb, - 0x3a, 0x3d, 0x39, 0x38, 0x6e, 0xee, 0x56, 0xef, 0xa0, 0xbb, 0xb0, 0xac, 0xa6, 0xf6, 0x5a, 0x47, - 0x2d, 0xb3, 0xd9, 0x95, 0xd5, 0x7b, 0xba, 0xa2, 0xeb, 0xe8, 0x1e, 0x20, 0x85, 0x75, 0x4e, 0x0f, - 0x0f, 0x9b, 0x66, 0xfb, 0x35, 0xc7, 0x33, 0x33, 0x2b, 0x7d, 0x76, 0xba, 0xd2, 0xe7, 0x66, 0x56, - 0xfa, 0xfc, 0xda, 0xd7, 0x19, 0x28, 0xc4, 0xf5, 0x72, 0xfc, 0x78, 0xd2, 0x92, 0x8f, 0xa7, 0xc9, - 0xb7, 0x9b, 0x3e, 0xe3, 0xed, 0x36, 0xfb, 0x95, 0x96, 0xb9, 0xee, 0x95, 0xb6, 0xa1, 0xde, 0x63, - 0x59, 0x21, 0xab, 0x07, 0x53, 0x95, 0xbb, 0xde, 0xb5, 0xfb, 0x54, 0x7e, 0x59, 0xca, 0xc7, 0x9a, - 0x01, 0x85, 0x88, 0xe2, 0xd0, 0xb5, 0x99, 0xad, 0xde, 0x7e, 0xa3, 0xb1, 0xf1, 0x57, 0x0d, 0x96, - 0x65, 0x9b, 0x7a, 0x19, 0x92, 0x41, 0x47, 0x7e, 0xf3, 0x1b, 0xbf, 0xba, 0xb5, 0x82, 0x12, 0x2f, - 0x2f, 0x3d, 0xf5, 0xf2, 0x1a, 0x77, 0xa0, 0x4c, 0xb2, 0x03, 0x19, 0x5b, 0x89, 0x04, 0xf9, 0x1c, - 0x0a, 0x71, 0xd1, 0x57, 0x35, 0x62, 0x79, 0xea, 0x34, 0xe6, 0x88, 0x62, 0x7c, 0x09, 0xc5, 0xd1, - 0xd1, 0x66, 0x7c, 0x13, 0xaf, 0x24, 0xbf, 0x89, 0x8b, 0x89, 0xef, 0xde, 0xb5, 0x2e, 0x94, 0x53, - 0x6d, 0x8d, 0x3f, 0x69, 0x45, 0x83, 0x95, 0xab, 0xc5, 0x6f, 0xb4, 0x01, 0xc5, 0x78, 0x27, 0x7e, - 0x4d, 0x99, 0xd9, 0xde, 0x8c, 0x39, 0x6b, 0x7f, 0xd7, 0x00, 0x8d, 0x53, 0x79, 0xd7, 0xa3, 0x43, - 0x9b, 0x39, 0xe7, 0xdf, 0xa8, 0x03, 0xdd, 0x85, 0x3c, 0xff, 0x76, 0x1a, 0x05, 0x2f, 0x77, 0x41, - 0x7a, 0x6d, 0x37, 0x15, 0x97, 0xcc, 0x8d, 0x71, 0x19, 0x75, 0xda, 0xec, 0x2d, 0x3a, 0xed, 0x67, - 0x67, 0x50, 0x49, 0xe3, 0xe8, 0x63, 0x78, 0xd0, 0x69, 0x1f, 0x9e, 0x1e, 0x34, 0xbb, 0xed, 0xe3, - 0x23, 0xeb, 0xf0, 0x78, 0xb7, 0x65, 0x9d, 0x1e, 0x75, 0x4e, 0x5a, 0x3b, 0xed, 0x97, 0xed, 0x16, - 0x4f, 0xb4, 0x1a, 0xac, 0x4c, 0x12, 0xba, 0xad, 0x5f, 0x76, 0xab, 0x9a, 0xc8, 0xce, 0x89, 0x99, - 0xe6, 0xe9, 0x6e, 0xfb, 0xb8, 0xaa, 0x37, 0xfe, 0x9c, 0x83, 0xa5, 0x26, 0xd7, 0xee, 0x78, 0x37, - 0xe4, 0xc2, 0x07, 0x52, 0x6b, 0xe9, 0xff, 0x08, 0x3e, 0xbd, 0xa6, 0x6e, 0x4b, 0x6e, 0x3d, 0xee, - 0xac, 0xdf, 0xb9, 0x89, 0xa6, 0x54, 0xf5, 0x4e, 0x83, 0x87, 0x71, 0x07, 0x1e, 0x11, 0x93, 0xbd, - 0x18, 0x35, 0xae, 0xb3, 0x34, 0xdd, 0xb7, 0x47, 0xbb, 0x7f, 0xf1, 0x8d, 0xd6, 0x28, 0x57, 0x7e, - 0x0d, 0xd5, 0x3d, 0xcc, 0xd2, 0xa7, 0x5d, 0xbb, 0xc6, 0xd0, 0x1e, 0x66, 0xa3, 0xcd, 0x9e, 0xcc, - 0xe5, 0x28, 0xe3, 0x16, 0x20, 0xde, 0x31, 0x53, 0x0c, 0x8a, 0xae, 0x5b, 0xca, 0xa9, 0x23, 0xfb, - 0x9f, 0xcc, 0x27, 0xa9, 0x0d, 0xf8, 0x75, 0x89, 0x8e, 0x72, 0xcb, 0xeb, 0x12, 0xdc, 0x9b, 0xaf, - 0x2b, 0xa6, 0xa9, 0x5d, 0x2e, 0xe1, 0xbe, 0x12, 0x85, 0xd2, 0x73, 0xa2, 0x10, 0xa1, 0x67, 0xd3, - 0xd5, 0x6d, 0xaa, 0x5a, 0x8d, 0x76, 0xfc, 0xde, 0xed, 0xc8, 0x72, 0xdf, 0xed, 0x97, 0xaf, 0x9f, - 0xf4, 0x3d, 0x76, 0x1e, 0xf5, 0xea, 0x0e, 0x19, 0xc4, 0xff, 0x7d, 0xca, 0xff, 0x37, 0x1d, 0xe2, - 0xc7, 0xc0, 0xdf, 0xf4, 0xf2, 0x81, 0x77, 0x89, 0x7f, 0x26, 0x1e, 0x81, 0x84, 0x91, 0xff, 0xe8, - 0x15, 0x35, 0x7e, 0xf1, 0x42, 0x00, 0xbd, 0xbc, 0x58, 0xf2, 0xc5, 0xff, 0x03, 0x00, 0x00, 0xff, - 0xff, 0xca, 0xe3, 0x43, 0x02, 0x6e, 0x15, 0x00, 0x00, + // 2895 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x59, 0xcb, 0x73, 0x1b, 0xc7, + 0xd1, 0x17, 0x9e, 0x04, 0x9a, 0x24, 0x08, 0x0e, 0x29, 0x6a, 0xb5, 0xb2, 0x2c, 0x8a, 0xb2, 0x2c, + 0x59, 0x0f, 0x48, 0x1f, 0x65, 0xc9, 0xa6, 0xfc, 0xa8, 0x0f, 0x02, 0x21, 0x09, 0x34, 0x49, 0xd1, + 0x0b, 0xd0, 0xfe, 0xec, 0xaf, 0xbe, 0xda, 0x5a, 0x2e, 0x86, 0xe0, 0x92, 0x8b, 0x9d, 0xf5, 0xcc, + 0x2c, 0x25, 0x1e, 0xbe, 0xbb, 0xab, 0x72, 0xca, 0x3f, 0x90, 0xca, 0xc5, 0x87, 0xe4, 0x9a, 0x6b, + 0xaa, 0x72, 0xce, 0x31, 0xf7, 0x9c, 0x72, 0xcc, 0x3d, 0x55, 0xb9, 0x25, 0x35, 0x8f, 0x5d, 0x2c, + 0x1e, 0x7c, 0x38, 0x55, 0xbe, 0x61, 0xbb, 0x7f, 0x3d, 0xd3, 0x33, 0xfd, 0xeb, 0x9e, 0x9e, 0x01, + 0xbc, 0xef, 0x7b, 0xc7, 0xf8, 0xc8, 0xe3, 0xb6, 0xd3, 0xc3, 0x01, 0xb7, 0x99, 0xd7, 0x8f, 0x7c, + 0x87, 0x7b, 0x24, 0xa8, 0x85, 0x94, 0x70, 0x82, 0xa6, 0xb4, 0xde, 0xbc, 0x29, 0x01, 0x8f, 0x46, + 0xe0, 0x98, 0xb1, 0x04, 0x6b, 0xde, 0xe8, 0x11, 0xd2, 0xf3, 0xf1, 0x23, 0xf9, 0xb5, 0x17, 0xed, + 0x3f, 0xe2, 0x5e, 0x1f, 0x33, 0xee, 0xf4, 0x43, 0x0d, 0xb8, 0x1a, 0x5b, 0xbb, 0x3e, 0x89, 0xba, + 0x6a, 0x0c, 0xad, 0x5a, 0x8c, 0x55, 0x7d, 0xd2, 0xc5, 0x3e, 0x53, 0xd2, 0x95, 0xbf, 0xe4, 0x60, + 0xb1, 0x9d, 0xb8, 0x64, 0x45, 0x41, 0x3b, 0xea, 0xf7, 0x1d, 0x7a, 0x82, 0x96, 0xa0, 0x18, 0x3a, + 0x8c, 0xe1, 0xae, 0x91, 0x59, 0xce, 0xdc, 0x2d, 0x58, 0xfa, 0x4b, 0xc8, 0xf7, 0x1d, 0xcf, 0xc7, + 0x5d, 0x23, 0xab, 0xe4, 0xea, 0x0b, 0x5d, 0x07, 0xe8, 0x11, 0x2f, 0xe8, 0xd9, 0x6f, 0xb1, 0xef, + 0x1b, 0xb9, 0xe5, 0xcc, 0xdd, 0xb2, 0x55, 0x96, 0x92, 0x6f, 0xb1, 0xef, 0x0b, 0x35, 0x27, 0xb6, + 0xd7, 0x0f, 0x29, 0x39, 0xc6, 0x46, 0x5e, 0xa9, 0x39, 0x69, 0x29, 0x01, 0xfa, 0x0c, 0x8a, 0x1e, + 0x63, 0x11, 0x66, 0x46, 0x61, 0x39, 0x77, 0x77, 0x7a, 0xf5, 0x56, 0x4d, 0x7b, 0x5b, 0x9b, 0xe4, + 0x5c, 0xad, 0x25, 0xb0, 0x96, 0x36, 0x41, 0x5f, 0xc3, 0x8c, 0x7b, 0xe0, 0x70, 0xfb, 0xc0, 0x63, + 0x9c, 0xd0, 0x13, 0xa3, 0x28, 0x87, 0xa8, 0x9d, 0x3d, 0x44, 0xe3, 0xc0, 0xe1, 0xaf, 0x95, 0x41, + 0x33, 0xe0, 0xf4, 0xc4, 0x9a, 0x76, 0x07, 0x12, 0xf3, 0x7b, 0xa8, 0x8e, 0x02, 0x50, 0x15, 0x72, + 0x47, 0xf8, 0x44, 0x6e, 0x47, 0xd9, 0x12, 0x3f, 0xd1, 0x63, 0x28, 0x1c, 0x3b, 0x7e, 0x84, 0xe5, + 0x56, 0x4c, 0xaf, 0x9a, 0xc9, 0x8c, 0x6a, 0xdf, 0xc5, 0x08, 0x0d, 0x12, 0x70, 0xfc, 0x8e, 0x5b, + 0x0a, 0xf8, 0x3c, 0xfb, 0x69, 0xc6, 0xb4, 0xa1, 0x20, 0xfd, 0x47, 0xcb, 0x30, 0xdd, 0xc5, 0xcc, + 0xa5, 0x5e, 0x28, 0x7c, 0xd3, 0x03, 0xa7, 0x45, 0xe8, 0x7d, 0x00, 0x16, 0xf5, 0x7a, 0x98, 0x49, + 0x40, 0x56, 0x02, 0x52, 0x12, 0xb4, 0x08, 0x05, 0xdf, 0xd9, 0xc3, 0xf1, 0x7e, 0xab, 0x8f, 0x95, + 0x3f, 0xdf, 0x87, 0xd9, 0xa1, 0x35, 0xa3, 0x0a, 0x64, 0xbd, 0xae, 0x9e, 0x20, 0xeb, 0xc9, 0x60, + 0x85, 0x94, 0x1c, 0x62, 0x97, 0xdb, 0x5e, 0x57, 0x8f, 0x5b, 0xd6, 0x92, 0x56, 0x17, 0x3d, 0x85, + 0x22, 0xe3, 0x0e, 0x8f, 0x98, 0x1c, 0xb7, 0xb2, 0x7a, 0x7d, 0xf2, 0x56, 0xd6, 0xda, 0x12, 0x64, + 0x69, 0x30, 0xba, 0x0f, 0xf3, 0x8a, 0xb4, 0xe9, 0x55, 0xa9, 0x50, 0x57, 0xa5, 0x62, 0x3d, 0xb5, + 0xb4, 0x45, 0x28, 0x60, 0x4a, 0x09, 0x35, 0x0a, 0xca, 0x75, 0xf9, 0x81, 0xd6, 0x00, 0x5c, 0x8a, + 0x1d, 0x8e, 0xbb, 0xb6, 0xc3, 0x8d, 0xa2, 0xde, 0x56, 0xc5, 0xfa, 0x5a, 0xcc, 0xfa, 0x5a, 0x27, + 0x66, 0xbd, 0x55, 0xd6, 0xe8, 0x3a, 0x47, 0x35, 0xc8, 0x1f, 0x92, 0x3d, 0x66, 0x4c, 0xc9, 0xe8, + 0x9b, 0xa7, 0xb8, 0xbc, 0x41, 0xf6, 0x2c, 0x89, 0x43, 0x9f, 0xc0, 0x14, 0x53, 0x5c, 0x30, 0x4a, + 0x72, 0x9e, 0xeb, 0x67, 0x12, 0xc6, 0x8a, 0xd1, 0x62, 0xf3, 0xd4, 0x32, 0x03, 0xa7, 0x8f, 0x8d, + 0xb2, 0xda, 0x3c, 0x29, 0xd9, 0x76, 0xfa, 0x18, 0x7d, 0x01, 0x15, 0xe6, 0xe2, 0xc0, 0xa1, 0x1e, + 0xb1, 0x7b, 0x94, 0x44, 0xa1, 0x01, 0x72, 0xf8, 0xa5, 0xc1, 0xf0, 0x5a, 0xfd, 0x4a, 0x68, 0xad, + 0x59, 0x96, 0xfe, 0x44, 0x4f, 0xa1, 0x84, 0x83, 0xae, 0x5a, 0xff, 0xf4, 0xb9, 0xeb, 0x9f, 0x92, + 0xd8, 0x3a, 0x47, 0xd7, 0xa0, 0x7c, 0x48, 0xf6, 0x6c, 0x97, 0x44, 0x01, 0x37, 0x66, 0x64, 0x66, + 0x96, 0x0e, 0xc9, 0x5e, 0x43, 0x7c, 0xa3, 0x9b, 0x30, 0xa3, 0xb2, 0x57, 0xeb, 0x67, 0xa5, 0x7e, + 0x5a, 0xc9, 0x12, 0x88, 0x4a, 0x64, 0x0d, 0xa9, 0x28, 0x88, 0x92, 0x29, 0xc8, 0x1d, 0x98, 0x0b, + 0xa2, 0x7e, 0xaa, 0x80, 0x31, 0x63, 0x4e, 0xa2, 0x2a, 0x41, 0xd4, 0x1f, 0x6c, 0x19, 0x43, 0xab, + 0x50, 0x88, 0x98, 0xd3, 0xc3, 0x46, 0x55, 0xfa, 0xff, 0xde, 0x29, 0xa1, 0xd8, 0x15, 0x18, 0x4b, + 0x41, 0x45, 0x2e, 0xb8, 0x24, 0x70, 0x23, 0x4a, 0x71, 0xe0, 0x9e, 0x18, 0xf3, 0x6a, 0xfa, 0x94, + 0x08, 0xdd, 0x87, 0xbc, 0xa8, 0x5c, 0x06, 0x92, 0x94, 0xbc, 0x32, 0x61, 0xd0, 0x2d, 0xd2, 0xc5, + 0x96, 0x04, 0x99, 0x7f, 0xbb, 0x0a, 0xb9, 0x0d, 0xb2, 0x37, 0x46, 0xfc, 0xb5, 0x84, 0xd9, 0x59, + 0x39, 0xcc, 0xcd, 0xd3, 0x69, 0x32, 0xca, 0xee, 0x15, 0x98, 0xf1, 0x02, 0xc6, 0x69, 0xe4, 0xaa, + 0xb5, 0xab, 0x94, 0x1b, 0x92, 0x0d, 0x48, 0x9d, 0x4f, 0x93, 0xfa, 0x21, 0x20, 0x45, 0x18, 0xfc, + 0x2e, 0xc4, 0x2e, 0xd7, 0x7b, 0xa7, 0x78, 0xaf, 0x32, 0xa6, 0x99, 0x52, 0x0c, 0x92, 0xba, 0x98, + 0x4a, 0x6a, 0x84, 0x20, 0xcf, 0x9d, 0x9e, 0xa2, 0x77, 0xd9, 0x92, 0xbf, 0x45, 0xd0, 0x29, 0x21, + 0x7d, 0x45, 0xc4, 0x92, 0x44, 0x97, 0x84, 0x40, 0xf2, 0x70, 0x0d, 0x80, 0x71, 0x87, 0xea, 0x54, + 0x82, 0xf3, 0x53, 0x49, 0xa3, 0xeb, 0xfc, 0x3f, 0xe5, 0xe0, 0x15, 0x98, 0x92, 0xee, 0x78, 0x5d, + 0xc9, 0xc0, 0xb2, 0x55, 0x14, 0x9f, 0xad, 0x2e, 0x7a, 0x16, 0x13, 0x62, 0x56, 0x0e, 0xb6, 0x7c, + 0xc6, 0xa6, 0x0f, 0x91, 0x62, 0x13, 0x66, 0x9d, 0xa8, 0xeb, 0x11, 0xbb, 0x8f, 0x39, 0xf5, 0x5c, + 0x26, 0x59, 0x39, 0xbd, 0x7a, 0xe7, 0x0c, 0xfb, 0xba, 0xc0, 0x6f, 0x29, 0xb8, 0x35, 0xe3, 0xa4, + 0xbe, 0xcc, 0xff, 0x83, 0x82, 0x1c, 0x1d, 0xdd, 0x85, 0xaa, 0xa8, 0xc9, 0x36, 0x8f, 0x68, 0xc0, + 0x34, 0xdf, 0xd5, 0x21, 0x57, 0x11, 0xf2, 0x8e, 0x10, 0x2b, 0xca, 0xdf, 0x83, 0x79, 0xe5, 0x40, + 0x1a, 0xaa, 0xce, 0xbd, 0x39, 0xa9, 0x18, 0x60, 0xcd, 0x7f, 0x2e, 0xc0, 0x4c, 0x7a, 0x76, 0x74, + 0x1b, 0x2a, 0x8e, 0xeb, 0x46, 0xd4, 0x71, 0x4f, 0x6c, 0xe6, 0x12, 0x8a, 0xe5, 0x24, 0x59, 0x6b, + 0x36, 0x96, 0xb6, 0x85, 0x10, 0x7d, 0x04, 0x55, 0xc1, 0x0b, 0xea, 0xe1, 0xc0, 0xc5, 0x1a, 0x98, + 0x95, 0xc0, 0xb9, 0x81, 0x5c, 0x41, 0xaf, 0xc0, 0x14, 0xe3, 0xdc, 0x7e, 0x8b, 0xa9, 0x71, 0x53, + 0x22, 0x8a, 0x8c, 0xf3, 0x6f, 0x31, 0x8d, 0x15, 0x2e, 0xa6, 0xc6, 0x4a, 0xa2, 0x68, 0x60, 0x8a, + 0x1e, 0xc3, 0xa2, 0x50, 0x1c, 0xe1, 0x13, 0x8e, 0x69, 0xdf, 0x8e, 0x67, 0x36, 0x6e, 0x49, 0x14, + 0x62, 0x9c, 0x7f, 0xa5, 0x54, 0x75, 0xad, 0x41, 0x5f, 0xc0, 0x35, 0x61, 0xc1, 0xa9, 0x13, 0x24, + 0xc5, 0xda, 0xf6, 0x1d, 0x2e, 0x72, 0xd0, 0x66, 0xc6, 0x5d, 0x69, 0x68, 0x30, 0xce, 0x3b, 0x69, + 0xc4, 0xa6, 0x02, 0xb4, 0xd1, 0x7b, 0x00, 0x9c, 0x33, 0x9b, 0xf3, 0xfd, 0x3d, 0x9b, 0x19, 0xf7, + 0x25, 0xba, 0xc4, 0x39, 0xeb, 0xf0, 0xfd, 0xbd, 0x36, 0x7a, 0x08, 0x0b, 0x42, 0xcb, 0x42, 0x8c, + 0xdd, 0x03, 0x9b, 0x3a, 0x1c, 0xdb, 0x6f, 0xc3, 0xbe, 0xf1, 0x50, 0xc2, 0xaa, 0x9c, 0xb3, 0xb6, + 0xd4, 0x58, 0x0e, 0xc7, 0xdf, 0x86, 0x7d, 0xb4, 0x0a, 0x97, 0x05, 0x3c, 0x70, 0x78, 0x44, 0x1d, + 0x3f, 0xc0, 0x8c, 0xe9, 0xfd, 0x59, 0x92, 0x06, 0x62, 0xac, 0xed, 0x81, 0x4e, 0xed, 0xd1, 0x3d, + 0x98, 0x17, 0x36, 0x21, 0x25, 0x8c, 0x74, 0xe3, 0x8d, 0xbf, 0xad, 0xf6, 0x93, 0x73, 0xb6, 0xa3, + 0xe4, 0x0a, 0xfb, 0x29, 0x18, 0x02, 0x8b, 0xdf, 0x85, 0x54, 0xb4, 0x59, 0xc7, 0x38, 0x35, 0xc5, + 0x87, 0xd2, 0x64, 0x89, 0x73, 0xd6, 0x1c, 0x52, 0x2b, 0x4b, 0xed, 0x19, 0x0e, 0xa2, 0xc0, 0xf5, + 0x24, 0x01, 0xb5, 0xd9, 0x72, 0xe2, 0x59, 0x73, 0xa0, 0x53, 0x36, 0x82, 0x76, 0x9c, 0xd9, 0x51, + 0xc8, 0x1d, 0xff, 0x48, 0xc3, 0xef, 0x48, 0x78, 0x85, 0x73, 0xb6, 0x2b, 0xc5, 0x49, 0x9c, 0x05, + 0x52, 0xc4, 0xf9, 0x03, 0x15, 0x4e, 0xce, 0x99, 0x88, 0xf3, 0x97, 0x70, 0x4d, 0x28, 0x5c, 0x12, + 0x1c, 0x63, 0xca, 0xe4, 0xd8, 0x8e, 0xef, 0xf1, 0x78, 0x99, 0x9f, 0x48, 0xf0, 0x55, 0xce, 0x59, + 0x63, 0x14, 0xa1, 0x06, 0xfe, 0x18, 0xc4, 0x82, 0xec, 0x03, 0xc7, 0xf7, 0x23, 0xd7, 0x0b, 0x94, + 0xe3, 0x22, 0x0c, 0xc6, 0xa7, 0xd2, 0x74, 0x91, 0x73, 0xf6, 0x3a, 0xad, 0x14, 0x91, 0x40, 0xff, + 0xa5, 0x16, 0xab, 0x32, 0xe1, 0x87, 0x48, 0xcd, 0xd8, 0x27, 0xcc, 0x58, 0x53, 0x2c, 0xe2, 0x9c, + 0x49, 0xe2, 0x7f, 0xad, 0x54, 0x5b, 0x84, 0x09, 0x16, 0x09, 0x93, 0x63, 0xe2, 0xb9, 0x58, 0xb8, + 0xcb, 0x3c, 0xa6, 0x19, 0x24, 0x1d, 0x7d, 0xae, 0x58, 0xc4, 0x39, 0xfb, 0x46, 0x20, 0x1a, 0x03, + 0x80, 0xf2, 0x73, 0x0d, 0xae, 0x2a, 0x9e, 0x90, 0x23, 0x1c, 0xd8, 0x0e, 0xe5, 0xde, 0xbe, 0xe3, + 0x72, 0x9d, 0x7f, 0x9f, 0xc9, 0xfc, 0x5b, 0x92, 0x6c, 0x11, 0xfa, 0xba, 0x56, 0xab, 0x94, 0x7d, + 0x0f, 0xc0, 0xf7, 0xfb, 0x82, 0x80, 0xdc, 0x66, 0xc6, 0x47, 0x8a, 0x80, 0xbe, 0xdf, 0xef, 0xf0, + 0x7d, 0xde, 0x4e, 0x69, 0x99, 0xcd, 0x8c, 0xc7, 0x69, 0x2d, 0x6b, 0xa3, 0x07, 0x80, 0xa4, 0xf6, + 0x80, 0x92, 0xa8, 0x77, 0x10, 0x46, 0xdc, 0xe6, 0x21, 0x33, 0xee, 0x29, 0x76, 0x0a, 0x54, 0xa2, + 0xe8, 0x84, 0x4c, 0x32, 0x2d, 0xa2, 0x81, 0xcd, 0x9d, 0x23, 0xd1, 0xf7, 0xaa, 0x95, 0x81, 0x66, + 0x5a, 0x44, 0x83, 0x8e, 0x94, 0xab, 0x05, 0x3d, 0x81, 0x25, 0x8a, 0x59, 0x48, 0x02, 0x86, 0x93, + 0x64, 0x0a, 0x9f, 0x3e, 0xb6, 0x99, 0xac, 0xaf, 0x59, 0x6b, 0x21, 0xd6, 0xea, 0x44, 0xda, 0x79, + 0xfa, 0xb8, 0x3d, 0xd9, 0x68, 0x4d, 0x18, 0xcd, 0x4c, 0x36, 0x5a, 0x3b, 0xd5, 0xe8, 0xa9, 0xcd, + 0x64, 0xf1, 0x9d, 0x64, 0xf4, 0xb4, 0x2d, 0x22, 0xac, 0x4e, 0xa8, 0x13, 0x0f, 0xfb, 0xdd, 0x54, + 0xba, 0x57, 0x54, 0x84, 0xa5, 0xf2, 0x3b, 0xa1, 0x4b, 0x12, 0xfd, 0x19, 0x5c, 0xc1, 0x84, 0xdb, + 0x7d, 0x8f, 0x85, 0x14, 0x77, 0x3d, 0x79, 0x00, 0xea, 0x00, 0xa9, 0xae, 0xe0, 0x32, 0x26, 0x7c, + 0x2b, 0xad, 0x55, 0xf1, 0xb9, 0x05, 0xb3, 0xe4, 0x18, 0x53, 0xdf, 0x09, 0x05, 0xf1, 0x3c, 0x22, + 0x9b, 0x84, 0xac, 0x35, 0xa3, 0x85, 0x96, 0x90, 0xa1, 0x0f, 0x61, 0x8e, 0x79, 0xbe, 0x2c, 0x88, + 0x9c, 0x70, 0xc7, 0xb7, 0x99, 0xec, 0x08, 0xb2, 0xd6, 0xac, 0x16, 0x77, 0x84, 0xb4, 0x2d, 0xd2, + 0xd0, 0x79, 0x7b, 0xf4, 0xd6, 0xa1, 0x5d, 0x3b, 0xc6, 0x2b, 0x17, 0x90, 0x74, 0x61, 0x41, 0x2b, + 0xdb, 0x4a, 0xa7, 0x1c, 0x78, 0x06, 0x57, 0xa2, 0xc0, 0x09, 0xd8, 0x5b, 0x4c, 0x45, 0xb7, 0xe3, + 0xf8, 0x3e, 0xa6, 0xaa, 0xbe, 0x1b, 0x0b, 0xca, 0xf1, 0x81, 0xba, 0x21, 0xb5, 0xb2, 0xc8, 0x8b, + 0x62, 0xb1, 0xef, 0xf8, 0x0c, 0xdb, 0x5e, 0xc0, 0x31, 0xa5, 0x51, 0x98, 0x5a, 0xf1, 0x15, 0x45, + 0x49, 0xa9, 0x6f, 0xa5, 0xd4, 0x6a, 0xc6, 0x37, 0xf0, 0xc1, 0x04, 0xcb, 0x28, 0xa0, 0xd8, 0x15, + 0xab, 0x4e, 0x7a, 0x2e, 0x43, 0x8e, 0x72, 0x73, 0x6c, 0x94, 0xdd, 0x01, 0x52, 0x0d, 0xf8, 0x10, + 0x16, 0x74, 0x43, 0xb1, 0x8a, 0x53, 0xc1, 0x7a, 0xa0, 0x88, 0xaa, 0x3a, 0x8a, 0x55, 0x9c, 0x84, + 0xea, 0x21, 0xa0, 0x09, 0x3e, 0x3f, 0x91, 0xb3, 0xcd, 0x7b, 0x63, 0xee, 0x7e, 0x09, 0xd7, 0x74, + 0x8f, 0x47, 0xa8, 0x8d, 0x1d, 0xea, 0x9f, 0xd8, 0xe2, 0x84, 0xd0, 0x15, 0xc1, 0xa8, 0x2d, 0x67, + 0xee, 0x96, 0xac, 0xab, 0x09, 0xa4, 0x29, 0x10, 0x9d, 0x01, 0x00, 0x7d, 0x0e, 0xe6, 0xc0, 0x5e, + 0x78, 0x37, 0x64, 0xfe, 0x48, 0x9a, 0x1b, 0x09, 0x42, 0x78, 0x99, 0xb6, 0xbe, 0x0f, 0xf3, 0xa2, + 0xeb, 0xf3, 0x58, 0xba, 0x18, 0x2f, 0xaa, 0x95, 0xa5, 0x14, 0x2a, 0xad, 0x6a, 0xb0, 0xd0, 0xc7, + 0x0e, 0x8b, 0xc4, 0x1e, 0xca, 0x5c, 0x54, 0x4b, 0x5b, 0x55, 0x4b, 0x8b, 0x55, 0x22, 0x7e, 0x6a, + 0x69, 0x0f, 0x00, 0x1d, 0x38, 0xcc, 0xa6, 0xb8, 0x4f, 0x38, 0x8e, 0xef, 0xd6, 0xc6, 0xc7, 0xd2, + 0xa5, 0xea, 0x81, 0xc3, 0x2c, 0xa9, 0x68, 0x2b, 0xb9, 0x68, 0x78, 0x75, 0xe3, 0x61, 0x8b, 0x62, + 0x2a, 0xa0, 0x4f, 0x65, 0x5f, 0x53, 0xd1, 0xe2, 0x6f, 0x94, 0x14, 0xdd, 0x80, 0xe9, 0xc3, 0xa8, + 0xdb, 0xc3, 0xea, 0x6a, 0x6d, 0x3c, 0x53, 0xf7, 0x34, 0x29, 0x12, 0x3d, 0xa9, 0x8f, 0x5a, 0x50, + 0x50, 0x0c, 0xfb, 0x5c, 0x5e, 0x4e, 0x9e, 0x5c, 0xb0, 0x81, 0xa9, 0x09, 0xcf, 0xe3, 0x66, 0x46, + 0x8d, 0x60, 0xfe, 0x2b, 0x0b, 0xd3, 0x29, 0xb1, 0xec, 0x0b, 0x23, 0x1a, 0xe8, 0x06, 0x46, 0xfe, + 0x16, 0xcb, 0x1c, 0xab, 0x01, 0x4c, 0x37, 0x15, 0xd5, 0x91, 0xfc, 0x6f, 0xa3, 0x65, 0x98, 0x51, + 0x6c, 0x72, 0x23, 0x6e, 0x7b, 0x81, 0x6c, 0x6c, 0x4b, 0x96, 0xba, 0xe3, 0x34, 0x22, 0xde, 0x0a, + 0xd2, 0x7d, 0x47, 0x7e, 0xa8, 0xef, 0x38, 0xa7, 0x59, 0x28, 0x9c, 0xdf, 0x2c, 0xa4, 0x6a, 0x75, + 0xf1, 0xcc, 0x5a, 0x0d, 0x23, 0xb5, 0x7a, 0xb8, 0xd1, 0x98, 0x1a, 0x6f, 0x34, 0x26, 0x65, 0x48, + 0xe9, 0x94, 0x0c, 0x99, 0x48, 0xba, 0xf2, 0x64, 0xd2, 0x6d, 0xe4, 0x4b, 0x97, 0xab, 0x4b, 0x1b, + 0xf9, 0xd2, 0xfb, 0xd5, 0x1b, 0x1b, 0xf9, 0xd2, 0x8d, 0xea, 0xf2, 0x0a, 0x81, 0xa2, 0xba, 0x2c, + 0x20, 0x04, 0x95, 0x76, 0xa7, 0xde, 0xd9, 0x6d, 0xdb, 0x3b, 0xcd, 0xed, 0xf5, 0xd6, 0xf6, 0xab, + 0xea, 0xa5, 0x94, 0xcc, 0xda, 0xdd, 0xde, 0x16, 0xb2, 0x0c, 0x5a, 0x84, 0xaa, 0x96, 0x35, 0xde, + 0x6c, 0xed, 0x6c, 0x36, 0x3b, 0xcd, 0xf5, 0x6a, 0x16, 0xcd, 0xc3, 0xac, 0x96, 0xbe, 0xac, 0xb7, + 0x36, 0x9b, 0xeb, 0xd5, 0x5c, 0x1a, 0x58, 0xdf, 0x6e, 0x34, 0x37, 0x85, 0x34, 0xbf, 0x91, 0x2f, + 0x95, 0xab, 0x60, 0xfe, 0x23, 0x07, 0xc5, 0x86, 0xbc, 0xff, 0x9a, 0x7f, 0xcd, 0xc2, 0x94, 0x85, + 0x7f, 0x88, 0x30, 0xe3, 0x23, 0x97, 0xfb, 0xcc, 0xe8, 0xe5, 0x7e, 0xf8, 0xfa, 0x9a, 0x1d, 0xbd, + 0xbe, 0x4e, 0xb8, 0xe5, 0xe5, 0x27, 0xde, 0xf2, 0x96, 0xa0, 0x48, 0x71, 0x4f, 0x24, 0x45, 0x51, + 0x37, 0xfb, 0xf2, 0x0b, 0xbd, 0x18, 0xbb, 0xff, 0x4e, 0x9d, 0x75, 0xff, 0x7d, 0x7d, 0x69, 0xe4, + 0x06, 0xfc, 0x63, 0x26, 0x83, 0x6e, 0x0f, 0xdf, 0x06, 0x45, 0xd8, 0x0a, 0xaf, 0x33, 0x43, 0xf7, + 0x41, 0x01, 0x8b, 0xaf, 0x84, 0xe5, 0x0b, 0x5c, 0x09, 0x5f, 0xcc, 0xc3, 0x9c, 0x3d, 0xec, 0xd8, + 0x8b, 0x0a, 0xcc, 0xd8, 0xa9, 0x21, 0x37, 0xf2, 0xa5, 0x5c, 0x35, 0x6f, 0x8d, 0x3f, 0x62, 0x98, + 0xbf, 0xca, 0x40, 0xc9, 0xd2, 0x79, 0x23, 0xce, 0xfd, 0xc1, 0xee, 0xd8, 0x34, 0x0a, 0x06, 0xdb, + 0x3c, 0xc7, 0xd2, 0xf9, 0xdc, 0xea, 0xa2, 0x36, 0x2c, 0x89, 0xe6, 0xd1, 0xeb, 0x05, 0xb8, 0x6b, + 0x87, 0x84, 0x71, 0x9b, 0xaa, 0x28, 0xe9, 0x27, 0xa3, 0xc1, 0x9b, 0xc3, 0x4e, 0x0c, 0xdb, 0x21, + 0x8c, 0xeb, 0x50, 0x5a, 0x8b, 0xe1, 0x04, 0xa9, 0xf9, 0xdb, 0x0c, 0x2c, 0x34, 0x48, 0xb0, 0xef, + 0xd1, 0x7e, 0x9b, 0x44, 0xd4, 0xc5, 0xbb, 0xa1, 0x4f, 0x9c, 0xae, 0xf9, 0xff, 0x17, 0xe6, 0xc0, + 0xc4, 0x25, 0x64, 0x27, 0x2f, 0xe1, 0x0e, 0xcc, 0xb9, 0xa4, 0x8b, 0x6d, 0x1c, 0x70, 0x7a, 0x12, + 0x12, 0x2f, 0xe0, 0xfa, 0xea, 0x5b, 0x11, 0xe2, 0x66, 0x22, 0x35, 0x61, 0xb0, 0x47, 0xe6, 0xaf, + 0x33, 0x90, 0x7b, 0x85, 0xb9, 0xd9, 0xf9, 0x25, 0x5c, 0x32, 0x3f, 0x4e, 0x45, 0xe3, 0x2e, 0xe4, + 0x68, 0xa4, 0xca, 0xdf, 0x10, 0xc7, 0xd2, 0x26, 0x96, 0x80, 0x98, 0x7f, 0xca, 0x42, 0x7e, 0xd3, + 0x63, 0xdc, 0xfc, 0x63, 0xe6, 0xc2, 0x5e, 0x3d, 0x1f, 0x79, 0x2f, 0x38, 0xfb, 0x25, 0xec, 0xf5, + 0xa5, 0xf8, 0xb5, 0x40, 0xb0, 0xf3, 0x73, 0x80, 0xd0, 0xe9, 0x89, 0x0e, 0xe6, 0x08, 0xab, 0xaa, + 0x3a, 0xbd, 0x6a, 0x24, 0xf6, 0x1d, 0x21, 0xdd, 0x71, 0x7a, 0xfa, 0xdc, 0x7b, 0x9d, 0xb1, 0xca, + 0x02, 0x2d, 0xc5, 0x3f, 0x66, 0x32, 0x2f, 0xca, 0x30, 0x65, 0xab, 0xb1, 0x5e, 0xcc, 0xc2, 0xb4, + 0x3d, 0x18, 0xc9, 0x7c, 0x37, 0xc4, 0xc5, 0x3c, 0x8d, 0x02, 0x66, 0x64, 0xe4, 0xb9, 0x72, 0xda, + 0xf2, 0x25, 0x06, 0xfd, 0x37, 0xcc, 0x05, 0xe2, 0xda, 0x9b, 0x72, 0x2a, 0x7b, 0xb6, 0x53, 0xd6, + 0xac, 0x30, 0xd8, 0x89, 0xdd, 0x32, 0x0f, 0xa1, 0xd8, 0x70, 0x02, 0x17, 0xfb, 0xbf, 0x50, 0x5c, + 0xd3, 0x0c, 0xfa, 0x65, 0x6f, 0xeb, 0x2b, 0x3f, 0x65, 0x92, 0x9a, 0x7d, 0x15, 0x2e, 0x0f, 0xd7, + 0x6c, 0x7b, 0x77, 0x67, 0xf3, 0x4d, 0x7d, 0xbd, 0x7a, 0x09, 0x5d, 0x86, 0x79, 0xad, 0x7a, 0xd5, + 0xdc, 0x6e, 0x5a, 0xf5, 0x8e, 0xaa, 0xde, 0xe3, 0x15, 0x3d, 0x8b, 0x96, 0x00, 0x69, 0x59, 0x7b, + 0x77, 0x6b, 0xab, 0x6e, 0xb5, 0xbe, 0x17, 0xf2, 0xdc, 0xc4, 0x4a, 0x9f, 0x1f, 0xaf, 0xf4, 0x85, + 0x89, 0x95, 0xbe, 0xb8, 0xf2, 0x53, 0x0e, 0x4a, 0x71, 0xbd, 0x1c, 0xbc, 0x0c, 0x65, 0xd2, 0x2f, + 0x43, 0xa3, 0x0f, 0x53, 0xd9, 0x09, 0x0f, 0x53, 0x93, 0x9f, 0xa0, 0x72, 0xa7, 0x3d, 0x41, 0x3d, + 0xd2, 0x8f, 0x4d, 0x79, 0x49, 0xab, 0x6b, 0x63, 0x95, 0xbb, 0xd6, 0x71, 0x7a, 0x4c, 0x3d, 0x9b, + 0xab, 0x97, 0x28, 0x13, 0x4a, 0x11, 0xc3, 0xb4, 0xeb, 0x70, 0x47, 0x3f, 0x6c, 0x25, 0xdf, 0xe6, + 0xef, 0x32, 0x30, 0xaf, 0x8e, 0xa9, 0x97, 0x94, 0xf4, 0x75, 0x73, 0x65, 0x7e, 0x77, 0x61, 0x06, + 0xa5, 0x9e, 0x95, 0xb2, 0x43, 0xcf, 0x4a, 0x83, 0x13, 0x28, 0x97, 0x3e, 0x81, 0xcc, 0xb5, 0x54, + 0x82, 0x3c, 0x84, 0x52, 0x5c, 0xf4, 0x75, 0x8d, 0x98, 0x1f, 0x5b, 0x8d, 0x95, 0x40, 0xcc, 0x4f, + 0xa0, 0x9c, 0x2c, 0x6d, 0xc2, 0x83, 0xff, 0x62, 0xfa, 0xc1, 0xbf, 0x9c, 0x7a, 0xd4, 0x5f, 0xe9, + 0xc0, 0xec, 0xd0, 0xb1, 0x26, 0xfa, 0x32, 0x79, 0xc0, 0x2a, 0x6b, 0xf9, 0x1b, 0x3d, 0x82, 0x72, + 0x3c, 0x93, 0x08, 0x53, 0x6e, 0xb2, 0x37, 0x03, 0xcc, 0xca, 0x1f, 0x32, 0x80, 0x06, 0xa9, 0xbc, + 0xee, 0xb1, 0xd0, 0xe1, 0xee, 0xc1, 0xcf, 0x3a, 0x81, 0x2e, 0x43, 0xf1, 0x90, 0xec, 0x0d, 0x36, + 0xaf, 0x70, 0x48, 0xf6, 0x5a, 0xdd, 0xa1, 0x7d, 0xc9, 0x9d, 0xbb, 0x2f, 0xc9, 0x49, 0x9b, 0xbf, + 0xc0, 0x49, 0x7b, 0x6f, 0x1f, 0x2a, 0xc3, 0x72, 0x74, 0x03, 0xae, 0xb5, 0x5b, 0x5b, 0xbb, 0x9b, + 0xf5, 0x4e, 0xeb, 0xcd, 0xb6, 0xbd, 0xf5, 0x66, 0xbd, 0x69, 0xef, 0x6e, 0xb7, 0x77, 0x9a, 0x8d, + 0xd6, 0xcb, 0x56, 0x53, 0x24, 0x9a, 0x01, 0x8b, 0xa3, 0x80, 0x4e, 0xf3, 0x7f, 0x3a, 0xd5, 0x8c, + 0xcc, 0xce, 0x11, 0x4d, 0x7d, 0x77, 0xbd, 0xf5, 0xa6, 0x9a, 0x5d, 0xfd, 0x4d, 0x01, 0xe6, 0xea, + 0x82, 0xbb, 0x83, 0xd9, 0x50, 0x17, 0x16, 0x14, 0xd7, 0x86, 0xff, 0x00, 0xb9, 0x7d, 0x4a, 0xdd, + 0x56, 0xd8, 0x5a, 0x7c, 0xb2, 0x7e, 0x78, 0x1e, 0x4c, 0xb3, 0xea, 0xc7, 0x0c, 0x5c, 0x8f, 0x4f, + 0xe0, 0x04, 0x98, 0x3e, 0x8b, 0xd1, 0xea, 0x69, 0x23, 0x8d, 0x9f, 0xdb, 0xc9, 0xec, 0x4f, 0x7e, + 0x96, 0x8d, 0x76, 0xe5, 0x7f, 0xa1, 0xfa, 0x0a, 0xf3, 0xe1, 0xd5, 0xae, 0x9c, 0x32, 0xd0, 0x2b, + 0xcc, 0x93, 0xc9, 0x6e, 0x9d, 0x89, 0xd1, 0x83, 0xdb, 0x80, 0xc4, 0x89, 0x39, 0x84, 0x60, 0xe8, + 0x34, 0x53, 0x01, 0x4d, 0xc6, 0xff, 0xe0, 0x6c, 0x90, 0x9e, 0x40, 0x84, 0x4b, 0x9e, 0x28, 0x17, + 0x0c, 0x97, 0xc4, 0x9e, 0x1f, 0xae, 0x18, 0xa6, 0x67, 0x39, 0x86, 0xab, 0x9a, 0x14, 0x9a, 0xcf, + 0xa9, 0x42, 0x84, 0xee, 0x8f, 0x57, 0xb7, 0xb1, 0x6a, 0x95, 0xcc, 0xf8, 0xe0, 0x62, 0x60, 0x35, + 0xef, 0x8b, 0x97, 0xdf, 0xdf, 0xea, 0x79, 0xfc, 0x20, 0xda, 0xab, 0xb9, 0xa4, 0x1f, 0xff, 0xb1, + 0xab, 0xfe, 0xbc, 0x75, 0x89, 0x1f, 0x0b, 0x7e, 0x9f, 0x9d, 0xdd, 0xf4, 0x8e, 0xf1, 0x57, 0xb2, + 0x09, 0x24, 0x9c, 0xfc, 0x3d, 0x5b, 0xd1, 0xdf, 0xcf, 0x9f, 0x4b, 0xc1, 0x5e, 0x51, 0x9a, 0x3c, + 0xf9, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd3, 0xe7, 0x77, 0x64, 0x4b, 0x1e, 0x00, 0x00, }