API Descriptions Update

.github/workflows/ci.yml

@@ -7,6 +7,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensures submodules are cloned
+
+
       - uses: actions/setup-node@v4
         with:
           node-version: "lts/*"

.github/workflows/deploy.yml

@@ -15,6 +15,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensures submodules are cloned
+
       - uses: actions/setup-node@v4
         with:
           node-version: "lts/*"

.github/workflows/test_typescript.yml

@@ -14,6 +14,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensures submodules are cloned
+
+
       - uses: actions/setup-node@v4
         with:
           node-version: "lts/*"

.github/workflows/update-core-deps.yml

@@ -12,6 +12,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: true  # Ensure submodules are checked out
+
+      - name: Update submodules
+        run: git submodule update --init --remote
       - uses: actions/setup-node@v4
         with:
           node-version: "lts/*"

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "inputfiles/mdn"]
+	path = inputfiles/mdn
+	url = https://github.com/mdn/content.git

src/build.ts

@@ -13,6 +13,7 @@ import { getInterfaceElementMergeData } from "./build/webref/elements.js";
 import { getInterfaceToEventMap } from "./build/webref/events.js";
 import { getWebidls } from "./build/webref/idl.js";
 import jsonc from "jsonc-parser";
+import { generateDescription } from "./build/mdn-comments.js";
 
 function mergeNamesakes(filtered: Browser.WebIdl) {
   const targets = [
@@ -90,33 +91,11 @@ async function emitDom() {
   const inputFolder = new URL("../inputfiles/", import.meta.url);
   const outputFolder = new URL("../generated/", import.meta.url);
 
-  // ${name} will be substituted with the name of an interface
-  const removeVerboseIntroductions: [RegExp, string][] = [
-    [
-      /^(The|A) ${name} interface of (the\s*)*((?:(?!API)[A-Za-z\d\s])+ API)/,
-      "This $3 interface ",
-    ],
-    [
-      /^(The|A) ${name} (interface|event|object) (is|represents|describes|defines)?/,
-      "",
-    ],
-    [
-      /^An object implementing the ${name} interface (is|represents|describes|defines)/,
-      "",
-    ],
-    [/^The ${name} is an interface representing/, ""],
-    [/^This type (is|represents|describes|defines)?/, ""],
-    [
-      /^The (((?:(?!API)[A-Za-z\s])+ API)) ${name} (represents|is|describes|defines)/,
-      "The $1 ",
-    ],
-  ];
-
   const overriddenItems = await readInputJSON("overridingTypes.jsonc");
   const addedItems = await readInputJSON("addedTypes.jsonc");
   const comments = await readInputJSON("comments.json");
   const deprecatedInfo = await readInputJSON("deprecatedMessage.json");
-  const documentationFromMDN = await readInputJSON("mdn/apiDescriptions.json");
+  const documentationFromMDN = await generateDescription();
   const removedItems = await readInputJSON("removedTypes.jsonc");
 
   async function readInputJSON(filename: string) {
@@ -159,8 +138,8 @@ async function emitDom() {
     );
     for (const [key, value] of Object.entries(descriptions)) {
       const target = idl.interfaces!.interface[key] || namespaces[key];
-      if (target && !value.startsWith("REDIRECT")) {
-        target.comment = transformVerbosity(key, value);
+      if (target) {
+        target.comment = value;
       }
     }
     return idl;
@@ -178,26 +157,12 @@ async function emitDom() {
     for (const [key, value] of Object.entries(descriptions)) {
       const target = idl.interfaces!.interface[key] || namespaces[key];
       if (target) {
-        target.deprecated = transformVerbosity(key, value);
+        target.deprecated = value;
       }
     }
     return idl;
   }
 
-  function transformVerbosity(name: string, description: string): string {
-    for (const regTemplate of removeVerboseIntroductions) {
-      const [{ source: template }, replace] = regTemplate;
-
-      const reg = new RegExp(template.replace(/\$\{name\}/g, name) + "\\s*");
-      const product = description.replace(reg, replace);
-      if (product !== description) {
-        return product.charAt(0).toUpperCase() + product.slice(1);
-      }
-    }
-
-    return description;
-  }
-
   /// Load the input file
   let webidl: Browser.WebIdl = {
     events: await getInterfaceToEventMap(),

src/build/mdn-comments.ts

@@ -0,0 +1,112 @@
+import fs from "fs/promises";
+import { basename } from "path";
+
+const basePath = new URL(
+  "../../inputfiles/mdn/files/en-us/web/api/",
+  import.meta.url,
+);
+
+function extractSummary(markdown: string): string {
+  // Remove frontmatter (--- at the beginning)
+  markdown = markdown.replace(/^---[\s\S]+?---\n/, "");
+
+  // Normalize line breaks by collapsing consecutive newlines into a single space
+  const normalizedText = markdown
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(
+      (line) =>
+        line &&
+        !line.startsWith("#") &&
+        !line.startsWith(">") &&
+        !line.startsWith("{{"),
+    )
+    .join(" ")
+    .replace(
+      /\{\{\s*(Glossary|HTMLElement|SVGAttr|SVGElement|cssxref|jsxref|HTTPHeader)\s*\(\s*["']((?:\\.|[^"\\])*?)["'].*?\)\s*\}\}/gi,
+      "$2",
+    ) // Extract first argument from multiple templates, handling escaped quotes & spaces
+    .replace(
+      /\{\{\s*domxref\s*\(\s*["']((?:\\.|[^"\\])*?)["'][^}]*\)\s*\}\}/gi,
+      "$1",
+    ) // Extract first argument from domxref, handling spaces
+    .replace(
+      /\{\{\s*(?:event|jsxref|cssref|specname)\s*\|\s*([^}]+)\s*\}\}/gi,
+      "$1",
+    ) // Handle event, jsxref, cssref, etc.
+    .replace(/\{\{\s*([^}]+)\s*\}\}/g, (_, match) => `[MISSING: ${match}]`) // Catch any remaining unhandled templates
+    .replace(/\[(.*?)\]\(.*?\)/g, "$1") // Keep link text but remove URLs
+    .replace(/\s+/g, " ") // Normalize spaces
+    .replace(/\n\s*/g, "\n") // Ensure line breaks are preserved
+    .replace(/"/g, "'")
+    .trim();
+
+  // Extract the first sentence (ending in . ! or ?)
+  const sentenceMatch = normalizedText.match(/(.*?[.!?])(?=\s|$)/);
+  if (sentenceMatch) {
+    return sentenceMatch[0]; // Return the first full sentence
+  }
+
+  return normalizedText.split(" ")[0] || ""; // Fallback: first word if no sentence found
+}
+
+async function getDirectories(dirPath: URL): Promise<URL[]> {
+  try {
+    const entries = await fs.readdir(dirPath, {
+      withFileTypes: true,
+    });
+    return entries
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => new URL(entry.name + "/", dirPath));
+  } catch (error) {
+    console.error("Error reading directories:", error);
+    return [];
+  }
+}
+
+async function getIndexMdContents(
+  folders: URL[],
+): Promise<{ [key: string]: string }> {
+  const results: { [key: string]: string } = {};
+
+  for (const folder of folders) {
+    const indexPath = new URL("index.md", folder);
+
+    try {
+      const content = await fs.readFile(indexPath, "utf-8");
+
+      // Improved title extraction
+      const titleMatch = content.match(/title:\s*["']?([^"'\n]+)["']?/);
+      const filename = basename(folder.toString());
+      const title = titleMatch
+        ? titleMatch[1].replace(/ extension$/, "")
+        : filename || "";
+
+      const summary = extractSummary(content);
+      results[title] = summary;
+    } catch (error) {
+      console.warn(`Skipping ${indexPath}: ${error}`);
+    }
+  }
+
+  return results;
+}
+
+export async function generateDescription(): Promise<Record<string, string>> {
+  const stats = await fs.stat(basePath);
+  if (!stats.isDirectory()) {
+    throw new Error(
+      "MDN submodule does not exist; try running `git submodule update --init`",
+    );
+  }
+  try {
+    const folders = await getDirectories(basePath);
+    if (folders.length > 0) {
+      return await getIndexMdContents(folders);
+    }
+  } catch (error) {
+    console.error("Error generating API descriptions:", error);
+  }
+
+  return {};
+}