(EAI-923) universal tagging system

packages/chatbot-server-mongodb-public/src/eval/ConversationEval.ts

@@ -21,7 +21,7 @@ import {
   Factuality,
 } from "autoevals";
 import { strict as assert } from "assert";
-import { MongoDbTag } from "../mongoDbMetadata";
+import { MongoDbTag } from "mongodb-rag-core";
 import { fuzzyLinkMatch } from "./fuzzyLinkMatch";
 import { binaryNdcgAtK } from "./scorers/binaryNdcgAtK";
 import { ConversationEvalCase as ConversationEvalCaseSource } from "mongodb-rag-core/eval";

packages/chatbot-server-mongodb-public/src/eval/bin/generateEvalCasesYamlFromCSV.ts

@@ -10,24 +10,26 @@
  Can be run through npm script or directly using node:
 
  ```bash
- npm run generate-eval-cases -- <csvFileName> <yamlFileName> [transformationType]
+ npm run generate-eval-cases -- <csvFileName> <yamlFileName> [transformationType] [transformationOptions]
  ```
 
  Or:
 
  ```bash
- node generateEvalCasesYamlFromCSV.js <csvFileName> <yamlFileName> [transformationType]
+ node generateEvalCasesYamlFromCSV.js <csvFileName> <yamlFileName> [transformationType] [transformationOptions]
  ```
 
  ### Arguments
 
  - `csvFilePath`: (Required) Absolute path to the input CSV file
  - `yamlFileName`: (Required) Name of the output YAML file (without .yml extension)
  - `transformationType`: (Optional) Type of transformation to apply to the cases
+ - `transformationOptions`: (Optional) Additional options for the transformation
 
  ### Available Transformations
 
- - `web`: Adds a "web" tag to all evaluation cases
+ - `addTags`: Adds specified tags to all evaluation cases
+ - `addCustomTags`: Adds specified custom tags to all evaluation cases
 
  ### File Paths
 
@@ -36,14 +38,14 @@
  ### Example
 
  ```bash
- npm run generate-eval-cases -- Users/first.lastname/Downloads/input-file.csv output-file-name web
+ npm run generate-eval-cases -- /path/to/input.csv output-name addTags tag1 tag2
  ```
 
  This will:
- 1. Read from: /Users/first.lastname/Downloads/input-file.csv
- 2. Apply the web transformation
- 3. Write to: evalCases/output-file-name.yml
- 4. Log missing resources to the console in a warning
+ 1. Read from: /path/to/input.csv
+ 2. Add tags "tag1" and "tag2" to all cases, after validating them against the MongoDbTags enum.
+ 3. Write to: evalCases/output-name.yml
+ 4. Log any missing resources to the console as warnings
 */
 
 import fs from "fs";
@@ -55,6 +57,7 @@ import {
 } from "mongodb-rag-core/eval";
 import { MONGODB_CONNECTION_URI, MONGODB_DATABASE_NAME } from "../../config";
 import { makeMongoDbPageStore } from "mongodb-rag-core";
+import { validateTags } from "mongodb-rag-core";
 
 const SRC_ROOT = path.resolve(__dirname, "../");
 
@@ -63,24 +66,30 @@ const pageStore = makeMongoDbPageStore({
   databaseName: MONGODB_DATABASE_NAME,
 });
 
-function addWebDataSourceTag(evalCases: ConversationEvalCase[]) {
-  return evalCases.map((caseItem) => {
-    const tags = caseItem.tags || [];
-    if (!tags.includes("web")) {
-      tags.push("web");
-    }
-    return {
-      ...caseItem,
-      tags,
-    };
-  });
+function addTags({
+  evalCases,
+  tagNames,
+  custom = false,
+}: {
+  evalCases: ConversationEvalCase[];
+  tagNames: string[];
+  custom?: boolean;
+}): ConversationEvalCase[] {
+  validateTags(tagNames, custom);
+  return evalCases.map((caseItem) => ({
+    ...caseItem,
+    tags: [...(caseItem.tags || []), ...tagNames],
+  }));
 }
 
 const transformationMap: Record<
   string,
-  (cases: ConversationEvalCase[]) => ConversationEvalCase[]
+  (cases: ConversationEvalCase[], options?: string[]) => ConversationEvalCase[]
 > = {
-  web: addWebDataSourceTag,
+  addTags: (cases: ConversationEvalCase[], options?: string[]) =>
+    addTags({ evalCases: cases, tagNames: options || [] }),
+  addCustomTags: (cases: ConversationEvalCase[], options?: string[]) =>
+    addTags({ evalCases: cases, tagNames: options || [], custom: true }),
   // Add more transformation functions here as needed
 };
 
@@ -103,15 +112,20 @@ async function main({
   csvFilePath,
   yamlFileName,
   transformationType,
+  transformationOptions,
 }: {
   csvFilePath: string;
   yamlFileName: string;
   transformationType?: keyof typeof transformationMap;
+  transformationOptions?: string[];
 }): Promise<void> {
   console.log(`Reading from: ${csvFilePath}`);
   const evalCases = await getConversationEvalCasesFromCSV(
     csvFilePath,
-    transformationType ? transformationMap[transformationType] : undefined
+    transformationType
+      ? (cases) =>
+          transformationMap[transformationType](cases, transformationOptions)
+      : undefined
   );
   const expectedUrls = Array.from(
     new Set(evalCases.flatMap((caseItem) => caseItem.expectedLinks ?? []))
@@ -142,15 +156,20 @@ async function main({
 // Checks if the script is being run directly (not imported as a module) and handles command-line arguments.
 if (require.main === module) {
   const args = process.argv.slice(2);
-  const [csvFilePath, yamlFileName, transformationType] = args;
+  const [
+    csvFilePath,
+    yamlFileName,
+    transformationType,
+    ...transformationOptions
+  ] = args;
   const availableTransformationTypes = Object.keys(transformationMap);
   if (
     args.length < 2 ||
     (transformationType &&
       !availableTransformationTypes.includes(transformationType))
   ) {
     console.error(
-      "Usage: node generateEvalCasesYamlFromCSV.js <csvFileName> <yamlFileName> [transformationType]\n" +
+      "Usage: node generateEvalCasesYamlFromCSV.js <csvFileName> <yamlFileName> [transformationType] [tranformationOptions]\n" +
         "Arguments:\n" +
         "  csvFileName: Input CSV file name (required)\n" +
         "  yamlFileName: Output YAML file name (required)\n" +
@@ -166,6 +185,7 @@ if (require.main === module) {
     csvFilePath,
     yamlFileName,
     transformationType,
+    transformationOptions,
   })
     .catch((error) => {
       console.error("Error:", error);

packages/chatbot-server-mongodb-public/src/eval/experiments/skillsQuestionsTest.eval.ts

@@ -17,7 +17,10 @@ async function conversationEval() {
   // Get dotcom question set eval cases from YAML
   const basePath = path.resolve(__dirname, "..", "..", "..", "evalCases");
   const conversationEvalCases = getConversationsEvalCasesFromYaml(
-    fs.readFileSync(path.resolve(basePath, "uni_skills_evaluation_questions.yml"), "utf8")
+    fs.readFileSync(
+      path.resolve(basePath, "uni_skills_evaluation_questions.yml"),
+      "utf8"
+    )
   );
 
   const generateConfig = {
@@ -52,4 +55,4 @@ async function conversationEval() {
     generate: generateConfig,
   });
 }
-conversationEval();
+conversationEval();

packages/chatbot-server-mongodb-public/src/lib.ts

@@ -3,4 +3,4 @@
   Export some modules from the implementation for use in things like evaluation.
  */
 export { systemPrompt } from "./systemPrompt";
-export * as mongoDbMetadata from "./mongoDbMetadata";
+export * as mongoDbMetadata from "mongodb-rag-core";

packages/chatbot-server-mongodb-public/src/processors/extractMongoDbMetadataFromUserMessage.eval.ts

@@ -4,7 +4,7 @@ import {
 } from "./extractMongoDbMetadataFromUserMessage";
 import { Eval } from "braintrust";
 import { Scorer } from "autoevals";
-import { MongoDbTag } from "../mongoDbMetadata";
+import { MongoDbTag } from "mongodb-rag-core";
 import {
   OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT,
   openAiClient,

packages/chatbot-server-mongodb-public/src/processors/extractMongoDbMetadataFromUserMessage.ts

@@ -8,7 +8,7 @@ import { OpenAI } from "mongodb-rag-core/openai";
 import {
   mongoDbProductNames,
   mongoDbProgrammingLanguageIds,
-} from "../mongoDbMetadata";
+} from "mongodb-rag-core";
 
 export const ExtractMongoDbMetadataFunctionSchema = z.object({
   programmingLanguage: z

packages/chatbot-server-mongodb-public/src/processors/makeStepBackUserQuery.eval.ts

@@ -6,7 +6,7 @@ import {
 } from "./makeStepBackUserQuery";
 import { Message, updateFrontMatter } from "mongodb-chatbot-server";
 import { ObjectId } from "mongodb-rag-core/mongodb";
-import { MongoDbTag } from "../mongoDbMetadata";
+import { MongoDbTag } from "mongodb-rag-core";
 import {
   OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT,
   OPENAI_API_KEY,

packages/chatbot-server-mongodb-public/src/processors/makeStepBackUserQuery.ts

@@ -110,8 +110,9 @@ const fewShotExamples: OpenAI.ChatCompletionMessageParam[] = [
       mongoDbProduct: "MongoDB University",
     })
   ),
-  makeAssistantFunctionCallMessage(name,{
-    transformedUserQuery: "What is the skill badge program on MongoDB University?",
+  makeAssistantFunctionCallMessage(name, {
+    transformedUserQuery:
+      "What is the skill badge program on MongoDB University?",
   } satisfies StepBackUserQueryMongoDbFunction),
 ];
 

packages/chatbot-server-mongodb-public/src/processors/retrieveRelevantContent.eval.ts

@@ -20,7 +20,7 @@ import { binaryNdcgAtK } from "../eval/scorers/binaryNdcgAtK";
 import { f1AtK } from "../eval/scorers/f1AtK";
 import { precisionAtK } from "../eval/scorers/precisionAtK";
 import { recallAtK } from "../eval/scorers/recallAtK";
-import { MongoDbTag } from "../mongoDbMetadata";
+import { MongoDbTag } from "mongodb-rag-core";
 import {
   extractMongoDbMetadataFromUserMessage,
   ExtractMongoDbMetadataFunction,

packages/chatbot-server-mongodb-public/src/processors/userMessageMongoDbGuardrail.eval.ts

@@ -5,7 +5,7 @@ import {
 } from "./userMessageMongoDbGuardrail";
 import { Eval } from "braintrust";
 import { Scorer, LLMClassifierFromTemplate } from "autoevals";
-import { MongoDbTag } from "../mongoDbMetadata";
+import { MongoDbTag } from "mongodb-rag-core";
 import {
   JUDGE_LLM,
   OPENAI_PREPROCESSOR_CHAT_COMPLETION_DEPLOYMENT,

packages/chatbot-server-mongodb-public/src/processors/userMessageMongoDbGuardrail.ts

@@ -147,9 +147,7 @@ const fewShotExamples: OpenAI.ChatCompletionMessageParam[] = [
     rejectMessage: false,
   } satisfies UserMessageMongoDbGuardrailFunction),
   // Example 16
-  makeUserMessage(
-    "What is a skill?"
-  ),
+  makeUserMessage("What is a skill?"),
   makeAssistantFunctionCallMessage(name, {
     reasoning:
       "This query is asking about MongoDB University's skills program, which allows users to earn a skill badge for taking a short course and completing an assessment. Therefore, it is relevant to MongoDB.",

packages/chatbot-server-mongodb-public/src/verifiedAnswers.eval.ts

@@ -1,5 +1,5 @@
 import { Eval, EvalCase, EvalScorer } from "braintrust";
-import { MongoDbTag } from "./mongoDbMetadata";
+import { MongoDbTag } from "mongodb-rag-core";
 import {
   findVerifiedAnswer,
   verifiedAnswerConfig,

packages/mongodb-rag-core/src/index.ts

@@ -25,3 +25,4 @@ export * from "./References";
 export * from "./VectorStore";
 export * from "./arrayFilters";
 export * from "./assertEnvVars";
+export * from "./mongoDbMetadata";

packages/chatbot-server-mongodb-public/src/mongoDbMetadata/products.ts → packages/mongodb-rag-core/src/mongoDbMetadata/products.ts

@@ -311,12 +311,14 @@ export const mongoDbProducts = [
   {
     id: "mongodb_university",
     name: "MongoDB University",
-    description: "Online platform that offers certifications, courses, labs, and skills badges",
+    description:
+      "Online platform that offers certifications, courses, labs, and skills badges",
   },
   {
     id: "skills",
     name: "MongoDB University Skills",
-    description: "An educational program that allows users to earn a skill badge after taking a short course and completing an assessment",
+    description:
+      "An educational program that allows users to earn a skill badge after taking a short course and completing an assessment",
   },
 ] as const satisfies MongoDbProduct[];
 

packages/mongodb-rag-core/src/mongoDbMetadata/tags.ts

@@ -0,0 +1,76 @@
+import { mongoDbProducts, mongodbDrivers } from "./products";
+import { mongoDbProgrammingLanguageIds } from "./programmingLanguages";
+import { mongoDbTopics } from "./topics";
+
+// Helpers for constructing the `MongoDbTag` union type
+const mongoDbProductIds = mongoDbProducts.map((product) => product.id);
+const mongoDbDriverIds = mongodbDrivers.map((driver) => driver.id);
+const mongoDbTopicIds = mongoDbTopics.map((topic) => topic.id);
+
+/**
+  All possible MongoDB tags. Useful for tagging evaluations.
+  */
+export type MongoDbTag =
+  | (typeof mongoDbProgrammingLanguageIds)[number]
+  | (typeof mongoDbProductIds)[number]
+  | (typeof mongoDbDriverIds)[number]
+  | (typeof mongoDbTopicIds)[number];
+
+/**
+  All possible MongoDB tags as enum.
+  */
+export const mongoDbTags = {
+  // Add all programming language IDs
+  ...mongoDbProgrammingLanguageIds.reduce((acc, id) => {
+    acc[id] = id;
+    return acc;
+  }, {} as Record<string, string>),
+
+  // Add all product IDs
+  ...mongoDbProductIds.reduce((acc, id) => {
+    acc[id] = id;
+    return acc;
+  }, {} as Record<string, string>),
+
+  // Add all driver IDs
+  ...mongoDbDriverIds.reduce((acc, id) => {
+    acc[id] = id;
+    return acc;
+  }, {} as Record<string, string>),
+
+  // Add all topic IDs
+  ...mongoDbTopicIds.reduce((acc, id) => {
+    acc[id] = id;
+    return acc;
+  }, {} as Record<string, string>),
+} as const;
+
+/**
+ Validates an array of tag names against the MongoDbTags enum.
+
+ @param tagNames - An array of strings representing tag names to validate
+ @param custom - A boolean flag indicating whether custom tags are allowed
+ @throws {Error} When non-custom tags are used that don't exist in MongoDbTags enum
+
+ @remarks
+ If custom is false, all tags must exist in the MongoDbTags enum.
+ If any invalid tags are found, throws an error with the list of invalid tags
+ and the allowed tags from MongoDbTags enum.
+ */
+export const validateTags = (tagNames: string[], custom: boolean): void => {
+  if (!custom) {
+    // check if all tags are allowed using the enum MongoDbTags
+    const invalidTags = tagNames.filter((tag) => !(tag in mongoDbTags));
+    if (invalidTags.length > 0) {
+      throw new Error(
+        `Invalid tags found: ${invalidTags.join(
+          ", "
+        )} \nUse the "addCustomTags" transformation instead or use allowed tags: \n  - ${Object.keys(
+          mongoDbTags
+        )
+          .sort()
+          .join("\n  - ")}`
+      );
+    }
+  }
+};