Add auto-link & various other url helpers (#2687)

frontend/package.json

@@ -83,6 +83,7 @@
     "tailwindcss": "^3.4.1",
     "terser-webpack-plugin": "^5.3.10",
     "thread-loader": "^4.0.4",
+    "tlds": "^1.259.0",
     "ts-loader": "^9.2.6",
     "tsconfig-paths-webpack-plugin": "^4.1.0",
     "type-fest": "^4.39.1",

frontend/src/components/ui/config-details.ts

@@ -24,6 +24,7 @@ import { isApiError } from "@/utils/api";
 import { DEPTH_SUPPORTED_SCOPES, isPageScopeType } from "@/utils/crawler";
 import { humanizeSchedule } from "@/utils/cron";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";
 import { getServerDefaults } from "@/utils/workflow";
 
 /**
@@ -312,7 +313,7 @@ export class ConfigDetails extends BtrixElement {
               crawlConfig?.description
                 ? html`
                     <p class="max-w-prose font-sans">
-                      ${crawlConfig.description}
+                      ${richText(crawlConfig.description)}
                     </p>
                   `
                 : undefined,

frontend/src/features/collections/collections-grid.ts

@@ -14,6 +14,7 @@ import { textSeparator } from "@/layouts/separator";
 import { RouteNamespace } from "@/routes";
 import { CollectionAccess, type PublicCollection } from "@/types/collection";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";
 
 /**
@@ -149,7 +150,7 @@ export class CollectionsGrid extends BtrixElement {
                     <p
                       class="mt-1.5 text-pretty leading-relaxed text-stone-500"
                     >
-                      ${collection.caption}
+                      ${richText(collection.caption, { shortenOnly: true })}
                     </p>
                   `}
                 </div>

frontend/src/pages/collections/collection.ts

@@ -11,6 +11,7 @@ import { page } from "@/layouts/page";
 import { RouteNamespace } from "@/routes";
 import type { PublicCollection } from "@/types/collection";
 import { formatRwpTimestamp } from "@/utils/replay";
+import { richText } from "@/utils/rich-text";
 
 enum Tab {
   Replay = "replay",
@@ -116,7 +117,9 @@ export class Collection extends BtrixElement {
 
     if (collection.caption) {
       header.secondary = html`
-        <div class="text-pretty text-stone-500">${collection.caption}</div>
+        <div class="text-pretty text-stone-500">
+          ${richText(collection.caption)}
+        </div>
       `;
     }
 

frontend/src/pages/org/archived-item-detail/archived-item-detail.ts

@@ -30,6 +30,7 @@ import {
 import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter";
 import { isArchivingDisabled } from "@/utils/orgs";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";
 
 import "./ui/qa";
@@ -901,7 +902,7 @@ export class ArchivedItemDetail extends BtrixElement {
                 this.item!.description?.length,
                 () =>
                   html`<pre class="whitespace-pre-line font-sans">
-${this.item?.description}
+                      ${richText(this.item?.description ?? "")}
                 </pre
                   >`,
                 () => noneText,

frontend/src/pages/org/browser-profiles-detail.ts

@@ -17,6 +17,7 @@ import { isApiError } from "@/utils/api";
 import { maxLengthValidator } from "@/utils/form";
 import { isArchivingDisabled } from "@/utils/orgs";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";
 
 const DESCRIPTION_MAXLENGTH = 500;
 
@@ -251,8 +252,9 @@ export class BrowserProfilesDetail extends BtrixElement {
         <div
           class="leading whitespace-pre-line rounded border p-5 leading-relaxed first-line:leading-[0]"
           >${this.profile
-            ? this.profile.description ||
+            ? this.profile.description
-              html`
+              ? richText(this.profile.description)
+              : html`
                   <div class="text-center text-neutral-400">
                     &nbsp;${msg("No description added.")}
                   </div>

frontend/src/pages/org/collection-detail.ts

@@ -38,6 +38,7 @@ import type { ArchivedItem, Crawl, Upload } from "@/types/crawler";
 import type { CrawlState } from "@/types/crawlState";
 import { pluralOf } from "@/utils/pluralize";
 import { formatRwpTimestamp } from "@/utils/replay";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";
 
 const ABORT_REASON_THROTTLE = "throttled";
@@ -203,7 +204,7 @@ export class CollectionDetail extends BtrixElement {
           ${this.collection
             ? this.collection.caption
               ? html`<div class="text-pretty text-neutral-600">
-                  ${this.collection.caption}
+                  ${richText(this.collection.caption)}
                 </div>`
               : html`<div
                   class="addSummary text-pretty rounded-md px-1 font-light text-neutral-500"

frontend/src/pages/org/dashboard.ts

@@ -24,8 +24,10 @@ import type { APIPaginatedList, APISortQuery } from "@/types/api";
 import { CollectionAccess, type Collection } from "@/types/collection";
 import { SortDirection } from "@/types/utils";
 import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";
 import { timeoutCache } from "@/utils/timeoutCache";
+import { toShortUrl } from "@/utils/url-helpers";
 import { cached } from "@/utils/weakCache";
 
 type Metrics = {
@@ -137,7 +139,9 @@ export class Dashboard extends BtrixElement {
           ${when(
             this.org?.publicDescription,
             (publicDescription) => html`
-              <div class="text-pretty text-stone-600">${publicDescription}</div>
+              <div class="text-pretty text-stone-600">
+                ${richText(publicDescription)}
+              </div>
             `,
           )}
           ${when(this.org?.publicUrl, (urlStr) => {
@@ -158,12 +162,12 @@ export class Dashboard extends BtrixElement {
                   label=${msg("Website")}
                 ></sl-icon>
                 <a
-                  class="font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
+                  class="truncate font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
                   href="${url.href}"
                   target="_blank"
                   rel="noopener noreferrer nofollow"
                 >
-                  ${url.href.split("//")[1].replace(/\/$/, "")}
+                  ${toShortUrl(url.href, null)}
                 </a>
               </div>
             `;

frontend/src/pages/public/org.ts

@@ -11,6 +11,8 @@ import type { APIPaginatedList, APISortQuery } from "@/types/api";
 import { CollectionAccess, type Collection } from "@/types/collection";
 import type { OrgData, PublicOrgCollections } from "@/types/org";
 import { SortDirection } from "@/types/utils";
+import { richText } from "@/utils/rich-text";
+import { toShortUrl } from "@/utils/url-helpers";
 
 @localized()
 @customElement("btrix-public-org")
@@ -152,7 +154,9 @@ export class PublicOrg extends BtrixElement {
             ${when(
               org.description,
               (description) => html`
-                <div class="text-pretty text-stone-600">${description}</div>
+                <div class="text-pretty text-stone-600">
+                  ${richText(description)}
+                </div>
               `,
             )}
             ${when(org.url, (urlStr) => {
@@ -173,12 +177,12 @@ export class PublicOrg extends BtrixElement {
                     label=${msg("Website")}
                   ></sl-icon>
                   <a
-                    class="font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
+                    class="truncate font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
                     href="${url.href}"
                     target="_blank"
                     rel="noopener noreferrer nofollow"
                   >
-                    ${url.href.split("//")[1].replace(/\/$/, "")}
+                    ${toShortUrl(url.href, null)}
                   </a>
                 </div>
               `;

frontend/src/stories/utils/RichText.mdx

@@ -0,0 +1,30 @@
+import {
+  Controls,
+  Primary,
+  Stories,
+  Title,
+} from "@storybook/addon-docs/blocks";
+
+import { Canvas, Meta } from "@storybook/addon-docs/blocks";
+
+import * as RichTextStories from "./RichText.stories";
+
+<Meta of={RichTextStories} />
+
+<Title />
+
+This is a rich text renderer that converts links in plain text into real links,
+in a similar way to the way social media posts often do. Links always open in a
+new tab, and the link detection is generally pretty forgiving.
+
+This should generally be used when displaying descriptions or other
+medium-length user-generated plain text, e.g. org or workflow descriptions.
+
+For longer text, consider using a more complete markdown setup, e.g. a
+Collection’s “About” section.
+
+<Primary />
+
+<Controls />
+
+<Stories />

frontend/src/stories/utils/RichText.stories.ts

@@ -0,0 +1,134 @@
+import type { Meta, StoryContext, StoryObj } from "@storybook/web-components";
+
+import { renderComponent, type RenderProps } from "./RichText";
+
+import { tw } from "@/utils/tailwind";
+
+const meta = {
+  title: "Utils/Rich Text",
+  render: renderComponent,
+  argTypes: {
+    options: {
+      name: "options",
+      description: "Optional options object, see below for details",
+    },
+    linkClass: {
+      name: "options.linkClass",
+      control: "text",
+      description: "CSS class to apply to links",
+      table: {
+        type: { summary: "string" },
+        defaultValue: {
+          summary:
+            "text-cyan-500 font-medium transition-colors hover:text-cyan-600",
+        },
+      },
+    },
+    maxLength: {
+      name: "options.maxLength",
+      control: {
+        type: "select",
+      },
+      // Hack: Storybook seems to convert null to undefined, so instead I'm using the string "null" and converting it back to null wherever it's used. See other places where this comment appears.
+      // -ESG
+      options: ["null", 5, 10, 15, 20],
+      description: "Maximum length of path portion of URLs",
+      table: {
+        type: { summary: "number | null" },
+        defaultValue: {
+          summary: "15",
+        },
+      },
+    },
+    shortenOnly: {
+      name: "options.shortenOnly",
+      control: {
+        type: "boolean",
+      },
+      description: "Whether to shorten URLs only",
+      table: {
+        type: { summary: "boolean" },
+        defaultValue: {
+          summary: "false",
+        },
+      },
+    },
+  },
+  args: {
+    content:
+      "Rich text example content with a link to https://example.com and a link without a protocol to webrecorder.net here. Long URLs like this one are cut short unless maxLength is overridden: https://webrecorder.net/blog/2025-05-28-create-use-and-automate-actions-with-custom-behaviors-in-browsertrix/#the-story-of-behaviors-in-browsertrix.",
+  },
+  parameters: {
+    docs: {
+      source: {
+        language: "typescript",
+        transform: (
+          code: string,
+          {
+            args: { content, linkClass, maxLength, shortenOnly },
+          }: StoryContext<RenderProps>,
+        ) =>
+          `import { richText } from "@/utils/rich-text";
+
+const content = ${JSON.stringify(content)};
+
+// Inside a Lit element, or wherever \`TemplateResult\`s are accepted:
+richText(content${
+            linkClass || maxLength || shortenOnly
+              ? `, { ${[
+                  linkClass && `linkClass: ${JSON.stringify(linkClass)}`,
+                  // Hack: convert "null" back to null (see above)
+                  // -ESG
+                  (typeof maxLength === "number" ||
+                    (maxLength as unknown as string) === "null") &&
+                    `maxLength: ${
+                      (maxLength as unknown as string) === "null"
+                        ? "null"
+                        : JSON.stringify(maxLength)
+                    }`,
+                  shortenOnly && `shortenOnly: ${JSON.stringify(shortenOnly)}`,
+                ]
+                  .filter(Boolean)
+                  .join(", ")} }`
+              : ``
+          });
+
+          `,
+      },
+    },
+  },
+} satisfies Meta<
+  RenderProps & {
+    options?: {
+      linkClass?: string;
+      maxLength?: number | null;
+      shortenOnly?: boolean;
+    };
+  }
+>;
+
+export default meta;
+type Story = StoryObj<RenderProps>;
+
+export const Basic: Story = {
+  args: {},
+};
+
+export const ShortenOnly: Story = {
+  args: {
+    shortenOnly: true,
+  },
+};
+
+export const MaxLength: Story = {
+  args: {
+    // Hack: use "null" instead of null (see above)
+    maxLength: "null" as unknown as null,
+  },
+};
+
+export const CustomLinkStyles: Story = {
+  args: {
+    linkClass: tw`rounded-md bg-purple-50 px-0.5 py-px italic text-purple-600 ring-1 ring-purple-300 hover:text-purple-800`,
+  },
+};

frontend/src/stories/utils/RichText.ts

@@ -0,0 +1,24 @@
+import { html } from "lit";
+
+import { richText } from "@/utils/rich-text";
+
+export type RenderProps = {
+  content: string;
+  linkClass?: string;
+  shortenOnly?: boolean;
+  maxLength?: number | null;
+};
+
+export const renderComponent = ({
+  content,
+  linkClass,
+  shortenOnly,
+  maxLength,
+}: RenderProps) => {
+  return html`${richText(content, {
+    linkClass,
+    shortenOnly,
+    // Hack: convert "null" back to null (see note in RichText.stories.ts)
+    maxLength: (maxLength as unknown as string) === "null" ? null : maxLength,
+  })}`;
+};

frontend/src/utils/rich-text.ts

@@ -0,0 +1,61 @@
+import { html } from "lit";
+import { guard } from "lit/directives/guard.js";
+
+import { definitelyUrl, detectLinks, toShortUrl } from "./url-helpers";
+
+/**
+ * This is a rich text renderer that converts links in plain text into real links, in a similar way to the way social media posts often do.
+ * Links always open in a new tab, and the link detection is generally pretty forgiving.
+ *
+ * This should generally be used when displaying descriptions or other medium-length user-generated plain text, e.g. org or workflow descriptions.
+ *
+ * For longer text, consider using a more complete markdown setup, e.g. a Collection’s “About” section.
+ *
+ * Options:
+ * - linkClass: The CSS class to apply to the links. Has some useful defaults, but can be overridden if necessary.
+ * - shortenOnly: Whether to only shorten the links, without converting them to real links. Useful when being used inside another link block (e.g. card links)
+ * - maxLength: The maximum length of path portion of the shortened URL. Defaults to 15 characters.
+ */
+export function richText(
+  content: string,
+  options: {
+    linkClass?: string;
+    shortenOnly?: boolean;
+    maxLength?: number | null;
+  } = {},
+) {
+  const {
+    shortenOnly,
+    linkClass = shortenOnly
+      ? "font-medium"
+      : "text-cyan-500 font-medium transition-colors hover:text-cyan-600",
+    maxLength = 15,
+  } = options;
+  const links = detectLinks(content);
+  return guard(
+    [content, linkClass, maxLength, shortenOnly],
+    () =>
+      html`${links.map((segment) => {
+        if (typeof segment === "string") {
+          return segment;
+        } else {
+          const url = definitelyUrl(segment.link);
+          if (!url) {
+            return segment.link;
+          }
+          if (shortenOnly) {
+            return html`<span class="${linkClass}" title="${url}"
+              >${toShortUrl(segment.link, maxLength)}</span
+            >`;
+          }
+          return html`<a
+            href="${url}"
+            target="_blank"
+            rel="noopener noreferrer"
+            class="${linkClass}"
+            >${toShortUrl(segment.link, maxLength)}</a
+          >`;
+        }
+      })}`,
+  );
+}

frontend/src/utils/url-helpers.ts

@@ -0,0 +1,115 @@
+// Adapted from https://github.com/bluesky-social/social-app/blob/main/src/lib/strings/url-helpers.ts and https://github.com/bluesky-social/social-app/blob/main/src/lib/strings/rich-text-detection.ts
+
+import TLDs from "tlds";
+
+export function isValidDomain(str: string): boolean {
+  return !!TLDs.find((tld) => {
+    const i = str.lastIndexOf(tld);
+    if (i === -1) {
+      return false;
+    }
+    return str.charAt(i - 1) === "." && i === str.length - tld.length;
+  });
+}
+
+/**
+ * Shortens a URL for use in rich text, etc. Remove protocol, trims "www." from the beginning of hosts, and trims pathname to a max length (configurable)
+ * @param url URL to shorten
+ * @param maxLength Max pathname length. Set to null to disable.
+ */
+export function toShortUrl(url: string, maxLength: number | null = 15): string {
+  try {
+    const urlp = new URL(url);
+    if (urlp.protocol !== "http:" && urlp.protocol !== "https:") {
+      return url;
+    }
+    const path =
+      (urlp.pathname === "/" ? "" : urlp.pathname) + urlp.search + urlp.hash;
+    if (maxLength && path.length > maxLength) {
+      return urlp.host + path.slice(0, maxLength - 2) + "...";
+    }
+    if (urlp.host.startsWith("www.")) {
+      return urlp.host.slice(4) + path;
+    }
+    return urlp.host + path;
+  } catch (e) {
+    return url;
+  }
+}
+
+// passes URL.parse, and has a TLD etc
+export function definitelyUrl(maybeUrl: string) {
+  try {
+    if (maybeUrl.endsWith(".")) return null;
+
+    // Prepend 'https://' if the input doesn't start with a protocol
+    if (!maybeUrl.startsWith("https://") && !maybeUrl.startsWith("http://")) {
+      maybeUrl = "https://" + maybeUrl;
+    }
+
+    const url = new URL(maybeUrl);
+
+    // Extract the hostname and split it into labels
+    const hostname = url.hostname;
+    const labels = hostname.split(".");
+
+    // Ensure there are at least two labels (e.g., 'example' and 'com')
+    if (labels.length < 2) return null;
+
+    const tld = labels[labels.length - 1];
+
+    // Check that the TLD is at least two characters long and contains only letters
+    if (!/^[a-z]{2,}$/i.test(tld)) return null;
+
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
+interface DetectedLink {
+  link: string;
+}
+type DetectedLinkable = string | DetectedLink;
+export function detectLinks(text: string): DetectedLinkable[] {
+  const re =
+    /((^|\s|\()@[a-z0-9.-]*)|((^|\s|\()https?:\/\/[\S]+)|((^|\s|\()(?<domain>[a-z][a-z0-9]*(\.[a-z0-9]+)+)[\S]*)/gi;
+  const segments = [];
+  let match;
+  let start = 0;
+  while ((match = re.exec(text))) {
+    let matchIndex = match.index;
+    let matchValue = match[0];
+
+    if (match.groups?.domain && !isValidDomain(match.groups.domain)) {
+      continue;
+    }
+
+    if (/\s|\(/.test(matchValue)) {
+      // HACK
+      // skip the starting space
+      // we have to do this because RN doesnt support negative lookaheads
+      // -prf
+      matchIndex++;
+      matchValue = matchValue.slice(1);
+    }
+
+    // strip ending punctuation
+    if (/[.,;!?]$/.test(matchValue)) {
+      matchValue = matchValue.slice(0, -1);
+    }
+    if (/[)]$/.test(matchValue) && !matchValue.includes("(")) {
+      matchValue = matchValue.slice(0, -1);
+    }
+
+    if (start !== matchIndex) {
+      segments.push(text.slice(start, matchIndex));
+    }
+    segments.push({ link: matchValue });
+    start = matchIndex + matchValue.length;
+  }
+  if (start < text.length) {
+    segments.push(text.slice(start));
+  }
+  return segments;
+}

frontend/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "outDir": "./dist/",
     "module": "esnext",
-    "target": "es6",
+    "target": "ES2018",
     "moduleResolution": "bundler",
     "allowJs": true,
     "strict": true,

frontend/web-test-runner.config.mjs

@@ -55,6 +55,7 @@ export default {
     }),
     esbuildPlugin({
       ts: true,
+      json: true,
       tsconfig: fileURLToPath(new URL("./tsconfig.json", import.meta.url)),
       target: "esnext",
       define: defineConfig,

frontend/yarn.lock

@@ -10659,6 +10659,11 @@ tinyspy@^3.0.0:
   resolved "https://registry.yarnpkg.com/tinyspy/-/tinyspy-3.0.2.tgz#86dd3cf3d737b15adcf17d7887c84a75201df20a"
   integrity sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==
 
+tlds@^1.259.0:
+  version "1.259.0"
+  resolved "https://registry.yarnpkg.com/tlds/-/tlds-1.259.0.tgz#f7e536e1fab65d7282443399417d317c309da3a3"
+  integrity sha512-AldGGlDP0PNgwppe2quAvuBl18UcjuNtOnDuUkqhd6ipPqrYYBt3aTxK1QTsBVknk97lS2JcafWMghjGWFtunw==
+
 tmp@^0.0.33:
   version "0.0.33"
   resolved "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz"