Skip to content

Commit ef88b26

Browse files
nlarewnlarew
committed
WIP - integrating traces + updated call signature
1 parent 135ed8b commit ef88b26

File tree

4 files changed

+102
-80
lines changed

4 files changed

+102
-80
lines changed

‎packages/chatbot-server-mongodb-public/src/config.ts

+36-21
Original file line numberDiff line numberDiff line change
@@ -252,29 +252,44 @@ export const config: AppConfig = {
252252
};
253253
},
254254
addMessageToConversationUpdateTrace:
255-
makeAddMessageToConversationUpdateTrace(
256-
retrievalConfig.findNearestNeighborsOptions.k,
257-
{ ...llmAsAJudgeConfig, percentToJudge: isProduction ? 0.1 : 1 }
258-
),
259-
rateMessageUpdateTrace: makeRateMessageUpdateTrace(llmAsAJudgeConfig),
260-
commentMessageUpdateTrace: makeCommentMessageUpdateTrace(
255+
makeAddMessageToConversationUpdateTrace({
256+
k: retrievalConfig.findNearestNeighborsOptions.k,
257+
llmAsAJudge: {
258+
...llmAsAJudgeConfig,
259+
percentToJudge: isProduction ? 0.1 : 1,
260+
},
261+
segment: {
262+
writeKey: "TODO",
263+
},
264+
}),
265+
rateMessageUpdateTrace: makeRateMessageUpdateTrace({
266+
llmAsAJudge: llmAsAJudgeConfig,
267+
segment: {
268+
writeKey: "TODO",
269+
},
270+
}),
271+
commentMessageUpdateTrace: makeCommentMessageUpdateTrace({
261272
openAiClient,
262-
JUDGE_LLM,
263-
SLACK_BOT_TOKEN !== undefined &&
273+
judgeLlm: JUDGE_LLM,
274+
slack:
275+
SLACK_BOT_TOKEN !== undefined &&
264276
SLACK_COMMENT_CONVERSATION_ID !== undefined
265-
? {
266-
token: SLACK_BOT_TOKEN,
267-
conversationId: SLACK_COMMENT_CONVERSATION_ID,
268-
llmAsAJudge: llmAsAJudgeConfig,
269-
braintrust: BRAINTRUST_CHATBOT_TRACING_PROJECT_NAME
270-
? {
271-
orgName: "mongodb-education-ai",
272-
projectName: BRAINTRUST_CHATBOT_TRACING_PROJECT_NAME,
273-
}
274-
: undefined,
275-
}
276-
: undefined
277-
),
277+
? {
278+
token: SLACK_BOT_TOKEN,
279+
conversationId: SLACK_COMMENT_CONVERSATION_ID,
280+
llmAsAJudge: llmAsAJudgeConfig,
281+
braintrust: BRAINTRUST_CHATBOT_TRACING_PROJECT_NAME
282+
? {
283+
orgName: "mongodb-education-ai",
284+
projectName: BRAINTRUST_CHATBOT_TRACING_PROJECT_NAME,
285+
}
286+
: undefined,
287+
}
288+
: undefined,
289+
segment: {
290+
writeKey: "TODO",
291+
},
292+
}),
278293
generateUserPrompt,
279294
systemPrompt,
280295
maxUserMessagesInConversation: 50,

‎packages/chatbot-server-mongodb-public/src/tracing/routesUpdateTraceHandlers.ts

+28-16
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,20 @@ import {
1515
TraceSegmentEventParams,
1616
} from "./segment";
1717

18-
export const makeAddMessageToConversationUpdateTrace: (
19-
k: number,
18+
export function makeAddMessageToConversationUpdateTrace({
19+
k,
20+
llmAsAJudge,
21+
segment,
22+
}: {
23+
k: number;
2024
llmAsAJudge?: LlmAsAJudge & {
2125
/**
2226
Percent of numbers to judge. Between 0 and 1.
2327
*/
2428
percentToJudge: number;
25-
},
26-
segment?: TraceSegmentEventParams
27-
) => UpdateTraceFunc = (k, llmAsAJudge, segment) => {
29+
};
30+
segment?: TraceSegmentEventParams;
31+
}): UpdateTraceFunc {
2832
validatePercentToJudge(llmAsAJudge?.percentToJudge);
2933

3034
const segmentTrackUserSentMessage = segment
@@ -99,7 +103,7 @@ export const makeAddMessageToConversationUpdateTrace: (
99103
},
100104
});
101105
};
102-
};
106+
}
103107

104108
function getTracingScores(
105109
tracingData: ReturnType<typeof extractTracingData>,
@@ -116,10 +120,13 @@ function getTracingScores(
116120
};
117121
}
118122

119-
export function makeRateMessageUpdateTrace(
120-
llmAsAJudge: LlmAsAJudge,
121-
segment?: TraceSegmentEventParams
122-
): UpdateTraceFunc {
123+
export function makeRateMessageUpdateTrace({
124+
llmAsAJudge,
125+
segment,
126+
}: {
127+
llmAsAJudge: LlmAsAJudge;
128+
segment?: TraceSegmentEventParams;
129+
}): UpdateTraceFunc {
123130
const segmentTrackUserRatedMessage = segment
124131
? makeTrackUserRatedMessage({
125132
writeKey: segment.writeKey,
@@ -155,9 +162,14 @@ export function makeRateMessageUpdateTrace(
155162
};
156163
}
157164

158-
export function makeCommentMessageUpdateTrace(
159-
openAiClient: OpenAI,
160-
judgeLlm: string,
165+
export function makeCommentMessageUpdateTrace({
166+
openAiClient,
167+
judgeLlm,
168+
slack,
169+
segment,
170+
}: {
171+
openAiClient: OpenAI;
172+
judgeLlm: string;
161173
slack?: {
162174
token: string;
163175
conversationId: string;
@@ -166,9 +178,9 @@ export function makeCommentMessageUpdateTrace(
166178
orgName: string;
167179
projectName: string;
168180
};
169-
},
170-
segment?: TraceSegmentEventParams
171-
): UpdateTraceFunc {
181+
};
182+
segment?: TraceSegmentEventParams;
183+
}): UpdateTraceFunc {
172184
const judgeMongoDbChatbotCommentSentiment =
173185
makeJudgeMongoDbChatbotCommentSentiment(openAiClient);
174186

‎packages/chatbot-server-mongodb-public/src/tracing/segment.test.ts

+8-24
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ describe("Segment Tracking", () => {
3131
it("should track user message with valid parameters", async () => {
3232
const trackUserSentMessage = makeTrackUserSentMessage({
3333
writeKey: "test-key",
34-
eventName: "User Sent Message",
3534
});
3635

3736
await trackUserSentMessage({
@@ -40,7 +39,7 @@ describe("Segment Tracking", () => {
4039
});
4140

4241
expect(mockAnalytics.track).toHaveBeenCalledWith({
43-
event: "User Sent Message",
42+
event: "AI Chat User Sent Message",
4443
userId: "user123",
4544
anonymousId: "anon123",
4645
timestamp: commonParams.createdAt.toISOString(),
@@ -58,7 +57,6 @@ describe("Segment Tracking", () => {
5857
it("should not track when userId or anonymousId is missing", async () => {
5958
const trackUserSentMessage = makeTrackUserSentMessage({
6059
writeKey: "test-key",
61-
eventName: "User Sent Message",
6260
});
6361

6462
await trackUserSentMessage({
@@ -73,7 +71,6 @@ describe("Segment Tracking", () => {
7371
it("should not track when origin URL is invalid", async () => {
7472
const trackUserSentMessage = makeTrackUserSentMessage({
7573
writeKey: "test-key",
76-
eventName: "User Sent Message",
7774
});
7875

7976
await trackUserSentMessage({
@@ -90,7 +87,6 @@ describe("Segment Tracking", () => {
9087
it("should track assistant response with verified answer", async () => {
9188
const trackAssistantResponded = makeTrackAssistantResponded({
9289
writeKey: "test-key",
93-
eventName: "Assistant Responded",
9490
});
9591

9692
await trackAssistantResponded({
@@ -99,7 +95,7 @@ describe("Segment Tracking", () => {
9995
});
10096

10197
expect(mockAnalytics.track).toHaveBeenCalledWith({
102-
event: "Assistant Responded",
98+
event: "AI Chat Assistant Responded",
10399
userId: "user123",
104100
anonymousId: "anon123",
105101
timestamp: commonParams.createdAt.toISOString(),
@@ -117,7 +113,6 @@ describe("Segment Tracking", () => {
117113
it("should track assistant response with rejection reason", async () => {
118114
const trackAssistantResponded = makeTrackAssistantResponded({
119115
writeKey: "test-key",
120-
eventName: "Assistant Responded",
121116
});
122117

123118
await trackAssistantResponded({
@@ -127,7 +122,7 @@ describe("Segment Tracking", () => {
127122
});
128123

129124
expect(mockAnalytics.track).toHaveBeenCalledWith({
130-
event: "Assistant Responded",
125+
event: "AI Chat Assistant Responded",
131126
userId: "user123",
132127
anonymousId: "anon123",
133128
timestamp: commonParams.createdAt.toISOString(),
@@ -146,7 +141,6 @@ describe("Segment Tracking", () => {
146141
it("should track assistant responses with valid parameters", async () => {
147142
const trackAssistantResponded = makeTrackAssistantResponded({
148143
writeKey: "test-key",
149-
eventName: "Assistant Responded",
150144
});
151145

152146
await trackAssistantResponded({
@@ -155,7 +149,7 @@ describe("Segment Tracking", () => {
155149
});
156150

157151
expect(mockAnalytics.track).toHaveBeenCalledWith({
158-
event: "Assistant Responded",
152+
event: "AI Chat Assistant Responded",
159153
userId: "user123",
160154
anonymousId: "anon123",
161155
timestamp: commonParams.createdAt.toISOString(),
@@ -173,7 +167,6 @@ describe("Segment Tracking", () => {
173167
it("should not track when userId or anonymousId is missing", async () => {
174168
const trackAssistantResponded = makeTrackAssistantResponded({
175169
writeKey: "test-key",
176-
eventName: "Assistant Responded",
177170
});
178171

179172
await trackAssistantResponded({
@@ -188,7 +181,6 @@ describe("Segment Tracking", () => {
188181
it("should not track when origin URL is invalid", async () => {
189182
const trackAssistantResponded = makeTrackAssistantResponded({
190183
writeKey: "test-key",
191-
eventName: "Assistant Responded",
192184
});
193185

194186
await trackAssistantResponded({
@@ -205,7 +197,6 @@ describe("Segment Tracking", () => {
205197
it("should track positive rating", async () => {
206198
const trackUserRatedMessage = makeTrackUserRatedMessage({
207199
writeKey: "test-key",
208-
eventName: "User Rated Message",
209200
});
210201

211202
await trackUserRatedMessage({
@@ -214,7 +205,7 @@ describe("Segment Tracking", () => {
214205
});
215206

216207
expect(mockAnalytics.track).toHaveBeenCalledWith({
217-
event: "User Rated Message",
208+
event: "AI Chat User Rated Message",
218209
userId: "user123",
219210
anonymousId: "anon123",
220211
timestamp: commonParams.createdAt.toISOString(),
@@ -232,7 +223,6 @@ describe("Segment Tracking", () => {
232223
it("should track negative rating", async () => {
233224
const trackUserRatedMessage = makeTrackUserRatedMessage({
234225
writeKey: "test-key",
235-
eventName: "User Rated Message",
236226
});
237227

238228
await trackUserRatedMessage({
@@ -241,7 +231,7 @@ describe("Segment Tracking", () => {
241231
});
242232

243233
expect(mockAnalytics.track).toHaveBeenCalledWith({
244-
event: "User Rated Message",
234+
event: "AI Chat User Rated Message",
245235
userId: "user123",
246236
anonymousId: "anon123",
247237
timestamp: commonParams.createdAt.toISOString(),
@@ -259,7 +249,6 @@ describe("Segment Tracking", () => {
259249
it("should not track when userId or anonymousId is missing", async () => {
260250
const trackUserRatedMessage = makeTrackUserRatedMessage({
261251
writeKey: "test-key",
262-
eventName: "User Rated Message",
263252
});
264253

265254
await trackUserRatedMessage({
@@ -274,7 +263,6 @@ describe("Segment Tracking", () => {
274263
it("should not track when origin URL is invalid", async () => {
275264
const trackUserRatedMessage = makeTrackUserRatedMessage({
276265
writeKey: "test-key",
277-
eventName: "User Rated Message",
278266
});
279267

280268
await trackUserRatedMessage({
@@ -291,7 +279,6 @@ describe("Segment Tracking", () => {
291279
it("should track user comment with positive rating", async () => {
292280
const trackUserCommentedMessage = makeTrackUserCommentedMessage({
293281
writeKey: "test-key",
294-
eventName: "User Commented Message",
295282
});
296283

297284
await trackUserCommentedMessage({
@@ -301,7 +288,7 @@ describe("Segment Tracking", () => {
301288
});
302289

303290
expect(mockAnalytics.track).toHaveBeenCalledWith({
304-
event: "User Commented Message",
291+
event: "AI Chat User Commented Message",
305292
userId: "user123",
306293
anonymousId: "anon123",
307294
timestamp: commonParams.createdAt.toISOString(),
@@ -320,7 +307,6 @@ describe("Segment Tracking", () => {
320307
it("should track user comment with negative rating", async () => {
321308
const trackUserCommentedMessage = makeTrackUserCommentedMessage({
322309
writeKey: "test-key",
323-
eventName: "User Commented Message",
324310
});
325311

326312
await trackUserCommentedMessage({
@@ -330,7 +316,7 @@ describe("Segment Tracking", () => {
330316
});
331317

332318
expect(mockAnalytics.track).toHaveBeenCalledWith({
333-
event: "User Commented Message",
319+
event: "AI Chat User Commented Message",
334320
userId: "user123",
335321
anonymousId: "anon123",
336322
timestamp: commonParams.createdAt.toISOString(),
@@ -349,7 +335,6 @@ describe("Segment Tracking", () => {
349335
it("should not track when userId or anonymousId is missing", async () => {
350336
const trackUserCommentedMessage = makeTrackUserCommentedMessage({
351337
writeKey: "test-key",
352-
eventName: "User Commented Message",
353338
});
354339

355340
await trackUserCommentedMessage({
@@ -365,7 +350,6 @@ describe("Segment Tracking", () => {
365350
it("should not track when origin URL is invalid", async () => {
366351
const trackUserCommentedMessage = makeTrackUserCommentedMessage({
367352
writeKey: "test-key",
368-
eventName: "User Commented Message",
369353
});
370354

371355
await trackUserCommentedMessage({

0 commit comments

Comments
 (0)