Add auto-link & various other url helpers (#2687)

2025-06-30 14:00:46 -04:00 · 2025-06-30 14:00:46 -04:00 · 52da39c2b4
commit 52da39c2b4
parent 0a68485c07
17 changed files with 406 additions and 18 deletions
--- a/frontend/package.json
+++ b/frontend/package.json
@ -83,6 +83,7 @@
    "tailwindcss": "^3.4.1",
    "terser-webpack-plugin": "^5.3.10",
    "thread-loader": "^4.0.4",
+    "tlds": "^1.259.0",
    "ts-loader": "^9.2.6",
    "tsconfig-paths-webpack-plugin": "^4.1.0",
    "type-fest": "^4.39.1",
--- a/frontend/src/components/ui/config-details.ts
+++ b/frontend/src/components/ui/config-details.ts
@ -24,6 +24,7 @@ import { isApiError } from "@/utils/api";
 import { DEPTH_SUPPORTED_SCOPES, isPageScopeType } from "@/utils/crawler";
 import { humanizeSchedule } from "@/utils/cron";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";
 import { getServerDefaults } from "@/utils/workflow";

 /**
@ -312,7 +313,7 @@ export class ConfigDetails extends BtrixElement {
              crawlConfig?.description
                ? html`
                    <p class="max-w-prose font-sans">
-                      ${crawlConfig.description}
+                      ${richText(crawlConfig.description)}
                    </p>
                  `
                : undefined,
--- a/frontend/src/features/collections/collections-grid.ts
+++ b/frontend/src/features/collections/collections-grid.ts
@ -14,6 +14,7 @@ import { textSeparator } from "@/layouts/separator";
 import { RouteNamespace } from "@/routes";
 import { CollectionAccess, type PublicCollection } from "@/types/collection";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";

 /**
@ -149,7 +150,7 @@ export class CollectionsGrid extends BtrixElement {
                    <p
                      class="mt-1.5 text-pretty leading-relaxed text-stone-500"
                    >
-                      ${collection.caption}
+                      ${richText(collection.caption, { shortenOnly: true })}
                    </p>
                  `}
                </div>
--- a/frontend/src/pages/collections/collection.ts
+++ b/frontend/src/pages/collections/collection.ts
@ -11,6 +11,7 @@ import { page } from "@/layouts/page";
 import { RouteNamespace } from "@/routes";
 import type { PublicCollection } from "@/types/collection";
 import { formatRwpTimestamp } from "@/utils/replay";
+import { richText } from "@/utils/rich-text";

 enum Tab {
  Replay = "replay",
@ -116,7 +117,9 @@ export class Collection extends BtrixElement {

    if (collection.caption) {
      header.secondary = html`
-        <div class="text-pretty text-stone-500">${collection.caption}</div>
+        <div class="text-pretty text-stone-500">
+          ${richText(collection.caption)}
+        </div>
      `;
    }

--- a/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts
+++ b/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts
@ -30,6 +30,7 @@ import {
 import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter";
 import { isArchivingDisabled } from "@/utils/orgs";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";

 import "./ui/qa";
@ -901,7 +902,7 @@ export class ArchivedItemDetail extends BtrixElement {
                this.item!.description?.length,
                () =>
                  html`<pre class="whitespace-pre-line font-sans">
-${this.item?.description}
+                      ${richText(this.item?.description ?? "")}
                </pre
                  >`,
                () => noneText,
--- a/frontend/src/pages/org/browser-profiles-detail.ts
+++ b/frontend/src/pages/org/browser-profiles-detail.ts
@ -17,6 +17,7 @@ import { isApiError } from "@/utils/api";
 import { maxLengthValidator } from "@/utils/form";
 import { isArchivingDisabled } from "@/utils/orgs";
 import { pluralOf } from "@/utils/pluralize";
+import { richText } from "@/utils/rich-text";

 const DESCRIPTION_MAXLENGTH = 500;

@ -251,12 +252,13 @@ export class BrowserProfilesDetail extends BtrixElement {
        <div
          class="leading whitespace-pre-line rounded border p-5 leading-relaxed first-line:leading-[0]"
          >${this.profile
-            ? this.profile.description ||
-              html`
-                <div class="text-center text-neutral-400">
-                  &nbsp;${msg("No description added.")}
-                </div>
-              `
+            ? this.profile.description
+              ? richText(this.profile.description)
+              : html`
+                  <div class="text-center text-neutral-400">
+                    &nbsp;${msg("No description added.")}
+                  </div>
+                `
            : nothing}</div
        >
      </section>
--- a/frontend/src/pages/org/collection-detail.ts
+++ b/frontend/src/pages/org/collection-detail.ts
@ -38,6 +38,7 @@ import type { ArchivedItem, Crawl, Upload } from "@/types/crawler";
 import type { CrawlState } from "@/types/crawlState";
 import { pluralOf } from "@/utils/pluralize";
 import { formatRwpTimestamp } from "@/utils/replay";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";

 const ABORT_REASON_THROTTLE = "throttled";
@ -203,7 +204,7 @@ export class CollectionDetail extends BtrixElement {
          ${this.collection
            ? this.collection.caption
              ? html`<div class="text-pretty text-neutral-600">
-                  ${this.collection.caption}
+                  ${richText(this.collection.caption)}
                </div>`
              : html`<div
                  class="addSummary text-pretty rounded-md px-1 font-light text-neutral-500"
--- a/frontend/src/pages/org/dashboard.ts
+++ b/frontend/src/pages/org/dashboard.ts
@ -24,8 +24,10 @@ import type { APIPaginatedList, APISortQuery } from "@/types/api";
 import { CollectionAccess, type Collection } from "@/types/collection";
 import { SortDirection } from "@/types/utils";
 import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter";
+import { richText } from "@/utils/rich-text";
 import { tw } from "@/utils/tailwind";
 import { timeoutCache } from "@/utils/timeoutCache";
+import { toShortUrl } from "@/utils/url-helpers";
 import { cached } from "@/utils/weakCache";

 type Metrics = {
@ -137,7 +139,9 @@ export class Dashboard extends BtrixElement {
          ${when(
            this.org?.publicDescription,
            (publicDescription) => html`
-              <div class="text-pretty text-stone-600">${publicDescription}</div>
+              <div class="text-pretty text-stone-600">
+                ${richText(publicDescription)}
+              </div>
            `,
          )}
          ${when(this.org?.publicUrl, (urlStr) => {
@ -158,12 +162,12 @@ export class Dashboard extends BtrixElement {
                  label=${msg("Website")}
                ></sl-icon>
                <a
-                  class="font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
+                  class="truncate font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
                  href="${url.href}"
                  target="_blank"
                  rel="noopener noreferrer nofollow"
                >
-                  ${url.href.split("//")[1].replace(/\/$/, "")}
+                  ${toShortUrl(url.href, null)}
                </a>
              </div>
            `;
--- a/frontend/src/pages/public/org.ts
+++ b/frontend/src/pages/public/org.ts
@ -11,6 +11,8 @@ import type { APIPaginatedList, APISortQuery } from "@/types/api";
 import { CollectionAccess, type Collection } from "@/types/collection";
 import type { OrgData, PublicOrgCollections } from "@/types/org";
 import { SortDirection } from "@/types/utils";
+import { richText } from "@/utils/rich-text";
+import { toShortUrl } from "@/utils/url-helpers";

@localized()
@customElement("btrix-public-org")
@ -152,7 +154,9 @@ export class PublicOrg extends BtrixElement {
            ${when(
              org.description,
              (description) => html`
-                <div class="text-pretty text-stone-600">${description}</div>
+                <div class="text-pretty text-stone-600">
+                  ${richText(description)}
+                </div>
              `,
            )}
            ${when(org.url, (urlStr) => {
@ -173,12 +177,12 @@ export class PublicOrg extends BtrixElement {
                    label=${msg("Website")}
                  ></sl-icon>
                  <a
-                    class="font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
+                    class="truncate font-medium leading-none text-stone-500 transition-colors hover:text-stone-600"
                    href="${url.href}"
                    target="_blank"
                    rel="noopener noreferrer nofollow"
                  >
-                    ${url.href.split("//")[1].replace(/\/$/, "")}
+                    ${toShortUrl(url.href, null)}
                  </a>
                </div>
              `;
--- a/frontend/src/stories/utils/RichText.mdx
+++ b/frontend/src/stories/utils/RichText.mdx
@ -0,0 +1,30 @@
+import {
+  Controls,
+  Primary,
+  Stories,
+  Title,
+} from "@storybook/addon-docs/blocks";
+
+import { Canvas, Meta } from "@storybook/addon-docs/blocks";
+
+import * as RichTextStories from "./RichText.stories";
+
+<Meta of={RichTextStories} />
+
+<Title />
+
+This is a rich text renderer that converts links in plain text into real links,
+in a similar way to the way social media posts often do. Links always open in a
+new tab, and the link detection is generally pretty forgiving.
+
+This should generally be used when displaying descriptions or other
+medium-length user-generated plain text, e.g. org or workflow descriptions.
+
+For longer text, consider using a more complete markdown setup, e.g. a
+Collection’s “About” section.
+
+<Primary />
+
+<Controls />
+
+<Stories />
--- a/frontend/src/stories/utils/RichText.stories.ts
+++ b/frontend/src/stories/utils/RichText.stories.ts
@ -0,0 +1,134 @@
+import type { Meta, StoryContext, StoryObj } from "@storybook/web-components";
+
+import { renderComponent, type RenderProps } from "./RichText";
+
+import { tw } from "@/utils/tailwind";
+
+const meta = {
+  title: "Utils/Rich Text",
+  render: renderComponent,
+  argTypes: {
+    options: {
+      name: "options",
+      description: "Optional options object, see below for details",
+    },
+    linkClass: {
+      name: "options.linkClass",
+      control: "text",
+      description: "CSS class to apply to links",
+      table: {
+        type: { summary: "string" },
+        defaultValue: {
+          summary:
+            "text-cyan-500 font-medium transition-colors hover:text-cyan-600",
+        },
+      },
+    },
+    maxLength: {
+      name: "options.maxLength",
+      control: {
+        type: "select",
+      },
+      // Hack: Storybook seems to convert null to undefined, so instead I'm using the string "null" and converting it back to null wherever it's used. See other places where this comment appears.
+      // -ESG
+      options: ["null", 5, 10, 15, 20],
+      description: "Maximum length of path portion of URLs",
+      table: {
+        type: { summary: "number | null" },
+        defaultValue: {
+          summary: "15",
+        },
+      },
+    },
+    shortenOnly: {
+      name: "options.shortenOnly",
+      control: {
+        type: "boolean",
+      },
+      description: "Whether to shorten URLs only",
+      table: {
+        type: { summary: "boolean" },
+        defaultValue: {
+          summary: "false",
+        },
+      },
+    },
+  },
+  args: {
+    content:
+      "Rich text example content with a link to https://example.com and a link without a protocol to webrecorder.net here. Long URLs like this one are cut short unless maxLength is overridden: https://webrecorder.net/blog/2025-05-28-create-use-and-automate-actions-with-custom-behaviors-in-browsertrix/#the-story-of-behaviors-in-browsertrix.",
+  },
+  parameters: {
+    docs: {
+      source: {
+        language: "typescript",
+        transform: (
+          code: string,
+          {
+            args: { content, linkClass, maxLength, shortenOnly },
+          }: StoryContext<RenderProps>,
+        ) =>
+          `import { richText } from "@/utils/rich-text";
+
+const content = ${JSON.stringify(content)};
+
+// Inside a Lit element, or wherever \`TemplateResult\`s are accepted:
+richText(content${
+            linkClass || maxLength || shortenOnly
+              ? `, { ${[
+                  linkClass && `linkClass: ${JSON.stringify(linkClass)}`,
+                  // Hack: convert "null" back to null (see above)
+                  // -ESG
+                  (typeof maxLength === "number" ||
+                    (maxLength as unknown as string) === "null") &&
+                    `maxLength: ${
+                      (maxLength as unknown as string) === "null"
+                        ? "null"
+                        : JSON.stringify(maxLength)
+                    }`,
+                  shortenOnly && `shortenOnly: ${JSON.stringify(shortenOnly)}`,
+                ]
+                  .filter(Boolean)
+                  .join(", ")} }`
+              : ``
+          });
+
+          `,
+      },
+    },
+  },
+} satisfies Meta<
+  RenderProps & {
+    options?: {
+      linkClass?: string;
+      maxLength?: number | null;
+      shortenOnly?: boolean;
+    };
+  }
+>;
+
+export default meta;
+type Story = StoryObj<RenderProps>;
+
+export const Basic: Story = {
+  args: {},
+};
+
+export const ShortenOnly: Story = {
+  args: {
+    shortenOnly: true,
+  },
+};
+
+export const MaxLength: Story = {
+  args: {
+    // Hack: use "null" instead of null (see above)
+    maxLength: "null" as unknown as null,
+  },
+};
+
+export const CustomLinkStyles: Story = {
+  args: {
+    linkClass: tw`rounded-md bg-purple-50 px-0.5 py-px italic text-purple-600 ring-1 ring-purple-300 hover:text-purple-800`,
+  },
+};
--- a/frontend/src/stories/utils/RichText.ts
+++ b/frontend/src/stories/utils/RichText.ts
@ -0,0 +1,24 @@
+import { html } from "lit";
+
+import { richText } from "@/utils/rich-text";
+
+export type RenderProps = {
+  content: string;
+  linkClass?: string;
+  shortenOnly?: boolean;
+  maxLength?: number | null;
+};
+
+export const renderComponent = ({
+  content,
+  linkClass,
+  shortenOnly,
+  maxLength,
+}: RenderProps) => {
+  return html`${richText(content, {
+    linkClass,
+    shortenOnly,
+    // Hack: convert "null" back to null (see note in RichText.stories.ts)
+    maxLength: (maxLength as unknown as string) === "null" ? null : maxLength,
+  })}`;
+};
--- a/frontend/src/utils/rich-text.ts
+++ b/frontend/src/utils/rich-text.ts
@ -0,0 +1,61 @@
+import { html } from "lit";
+import { guard } from "lit/directives/guard.js";
+
+import { definitelyUrl, detectLinks, toShortUrl } from "./url-helpers";
+
+/**
+ * This is a rich text renderer that converts links in plain text into real links, in a similar way to the way social media posts often do.
+ * Links always open in a new tab, and the link detection is generally pretty forgiving.
+ *
+ * This should generally be used when displaying descriptions or other medium-length user-generated plain text, e.g. org or workflow descriptions.
+ *
+ * For longer text, consider using a more complete markdown setup, e.g. a Collection’s “About” section.
+ *
+ * Options:
+ * - linkClass: The CSS class to apply to the links. Has some useful defaults, but can be overridden if necessary.
+ * - shortenOnly: Whether to only shorten the links, without converting them to real links. Useful when being used inside another link block (e.g. card links)
+ * - maxLength: The maximum length of path portion of the shortened URL. Defaults to 15 characters.
+ */
+export function richText(
+  content: string,
+  options: {
+    linkClass?: string;
+    shortenOnly?: boolean;
+    maxLength?: number | null;
+  } = {},
+) {
+  const {
+    shortenOnly,
+    linkClass = shortenOnly
+      ? "font-medium"
+      : "text-cyan-500 font-medium transition-colors hover:text-cyan-600",
+    maxLength = 15,
+  } = options;
+  const links = detectLinks(content);
+  return guard(
+    [content, linkClass, maxLength, shortenOnly],
+    () =>
+      html`${links.map((segment) => {
+        if (typeof segment === "string") {
+          return segment;
+        } else {
+          const url = definitelyUrl(segment.link);
+          if (!url) {
+            return segment.link;
+          }
+          if (shortenOnly) {
+            return html`<span class="${linkClass}" title="${url}"
+              >${toShortUrl(segment.link, maxLength)}</span
+            >`;
+          }
+          return html`<a
+            href="${url}"
+            target="_blank"
+            rel="noopener noreferrer"
+            class="${linkClass}"
+            >${toShortUrl(segment.link, maxLength)}</a
+          >`;
+        }
+      })}`,
+  );
+}
--- a/frontend/src/utils/url-helpers.ts
+++ b/frontend/src/utils/url-helpers.ts
@ -0,0 +1,115 @@
+// Adapted from https://github.com/bluesky-social/social-app/blob/main/src/lib/strings/url-helpers.ts and https://github.com/bluesky-social/social-app/blob/main/src/lib/strings/rich-text-detection.ts
+
+import TLDs from "tlds";
+
+export function isValidDomain(str: string): boolean {
+  return !!TLDs.find((tld) => {
+    const i = str.lastIndexOf(tld);
+    if (i === -1) {
+      return false;
+    }
+    return str.charAt(i - 1) === "." && i === str.length - tld.length;
+  });
+}
+
+/**
+ * Shortens a URL for use in rich text, etc. Remove protocol, trims "www." from the beginning of hosts, and trims pathname to a max length (configurable)
+ * @param url URL to shorten
+ * @param maxLength Max pathname length. Set to null to disable.
+ */
+export function toShortUrl(url: string, maxLength: number | null = 15): string {
+  try {
+    const urlp = new URL(url);
+    if (urlp.protocol !== "http:" && urlp.protocol !== "https:") {
+      return url;
+    }
+    const path =
+      (urlp.pathname === "/" ? "" : urlp.pathname) + urlp.search + urlp.hash;
+    if (maxLength && path.length > maxLength) {
+      return urlp.host + path.slice(0, maxLength - 2) + "...";
+    }
+    if (urlp.host.startsWith("www.")) {
+      return urlp.host.slice(4) + path;
+    }
+    return urlp.host + path;
+  } catch (e) {
+    return url;
+  }
+}
+
+// passes URL.parse, and has a TLD etc
+export function definitelyUrl(maybeUrl: string) {
+  try {
+    if (maybeUrl.endsWith(".")) return null;
+
+    // Prepend 'https://' if the input doesn't start with a protocol
+    if (!maybeUrl.startsWith("https://") && !maybeUrl.startsWith("http://")) {
+      maybeUrl = "https://" + maybeUrl;
+    }
+
+    const url = new URL(maybeUrl);
+
+    // Extract the hostname and split it into labels
+    const hostname = url.hostname;
+    const labels = hostname.split(".");
+
+    // Ensure there are at least two labels (e.g., 'example' and 'com')
+    if (labels.length < 2) return null;
+
+    const tld = labels[labels.length - 1];
+
+    // Check that the TLD is at least two characters long and contains only letters
+    if (!/^[a-z]{2,}$/i.test(tld)) return null;
+
+    return url.toString();
+  } catch {
+    return null;
+  }
+}
+
+interface DetectedLink {
+  link: string;
+}
+type DetectedLinkable = string | DetectedLink;
+export function detectLinks(text: string): DetectedLinkable[] {
+  const re =
+    /((^|\s|\()@[a-z0-9.-]*)|((^|\s|\()https?:\/\/[\S]+)|((^|\s|\()(?<domain>[a-z][a-z0-9]*(\.[a-z0-9]+)+)[\S]*)/gi;
+  const segments = [];
+  let match;
+  let start = 0;
+  while ((match = re.exec(text))) {
+    let matchIndex = match.index;
+    let matchValue = match[0];
+
+    if (match.groups?.domain && !isValidDomain(match.groups.domain)) {
+      continue;
+    }
+
+    if (/\s|\(/.test(matchValue)) {
+      // HACK
+      // skip the starting space
+      // we have to do this because RN doesnt support negative lookaheads
+      // -prf
+      matchIndex++;
+      matchValue = matchValue.slice(1);
+    }
+
+    // strip ending punctuation
+    if (/[.,;!?]$/.test(matchValue)) {
+      matchValue = matchValue.slice(0, -1);
+    }
+    if (/[)]$/.test(matchValue) && !matchValue.includes("(")) {
+      matchValue = matchValue.slice(0, -1);
+    }
+
+    if (start !== matchIndex) {
+      segments.push(text.slice(start, matchIndex));
+    }
+    segments.push({ link: matchValue });
+    start = matchIndex + matchValue.length;
+  }
+  if (start < text.length) {
+    segments.push(text.slice(start));
+  }
+  return segments;
+}
--- a/frontend/tsconfig.json
+++ b/frontend/tsconfig.json
@ -2,7 +2,7 @@
  "compilerOptions": {
    "outDir": "./dist/",
    "module": "esnext",
-    "target": "es6",
+    "target": "ES2018",
    "moduleResolution": "bundler",
    "allowJs": true,
    "strict": true,
--- a/frontend/web-test-runner.config.mjs
+++ b/frontend/web-test-runner.config.mjs
@ -55,6 +55,7 @@ export default {
    }),
    esbuildPlugin({
      ts: true,
+      json: true,
      tsconfig: fileURLToPath(new URL("./tsconfig.json", import.meta.url)),
      target: "esnext",
      define: defineConfig,
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@ -10659,6 +10659,11 @@ tinyspy@^3.0.0:
  resolved "https://registry.yarnpkg.com/tinyspy/-/tinyspy-3.0.2.tgz#86dd3cf3d737b15adcf17d7887c84a75201df20a"
  integrity sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==

+tlds@^1.259.0:
+  version "1.259.0"
+  resolved "https://registry.yarnpkg.com/tlds/-/tlds-1.259.0.tgz#f7e536e1fab65d7282443399417d317c309da3a3"
+  integrity sha512-AldGGlDP0PNgwppe2quAvuBl18UcjuNtOnDuUkqhd6ipPqrYYBt3aTxK1QTsBVknk97lS2JcafWMghjGWFtunw==
+
 tmp@^0.0.33:
  version "0.0.33"
  resolved "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz"