fix: Crawler proxy selection fixes (#2280)

- Hides proxy form control if there are no proxy servers available
- Fixes org default proxy value not being saved

frontend/package.json

@@ -8,6 +8,7 @@
     "@formatjs/intl-durationformat": "^0.6.4",
     "@formatjs/intl-localematcher": "^0.5.9",
     "@ianvs/prettier-plugin-sort-imports": "^4.2.1",
+    "@lit/context": "^1.1.3",
     "@lit/localize": "^0.12.1",
     "@lit/task": "^1.0.0",
     "@novnc/novnc": "^1.4.0-beta",

frontend/src/components/ui/select-crawler-proxy.ts

@@ -1,10 +1,11 @@
 import { localized, msg } from "@lit/localize";
-import { type SlSelect } from "@shoelace-style/shoelace";
+import type { SlSelect } from "@shoelace-style/shoelace";
 import { html } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
+import { ifDefined } from "lit/directives/if-defined.js";
 
-import type { ProxiesAPIResponse, Proxy } from "@/pages/org/types";
-import LiteElement from "@/utils/LiteElement";
+import { BtrixElement } from "@/classes/BtrixElement";
+import type { Proxy } from "@/pages/org/types";
 
 type SelectCrawlerProxyChangeDetail = {
   value: string | null;
@@ -26,30 +27,40 @@ export type SelectCrawlerProxyUpdateEvent =
  * Usage example:
  * ```ts
  * <btrix-select-crawler-proxy
- *   orgId=${orgId}
- *   on-change=${({value}) => selectedcrawlerProxy = value}
+ *   .proxyServers=${proxyServers}
+ *   btrix-change=${({value}) => selectedcrawlerProxy = value}
  * ></btrix-select-crawler-proxy>
  * ```
  *
- * @event on-change
+ * @fires btrix-change
  */
 @customElement("btrix-select-crawler-proxy")
 @localized()
-export class SelectCrawlerProxy extends LiteElement {
+export class SelectCrawlerProxy extends BtrixElement {
+  @property({ type: String })
+  defaultProxyId: string | null = null;
+
+  @property({ type: Array })
+  proxyServers: Proxy[] = [];
+
   @property({ type: String })
   proxyId: string | null = null;
 
+  @property({ type: String })
+  size?: SlSelect["size"];
+
   @state()
   private selectedProxy?: Proxy;
 
   @state()
   private defaultProxy?: Proxy;
 
-  @state()
-  private allProxies?: Proxy[];
+  public get value() {
+    return this.selectedProxy?.id || "";
+  }
 
   protected firstUpdated() {
-    void this.fetchOrgProxies();
+    void this.initProxies();
   }
   // credit: https://dev.to/jorik/country-code-to-flag-emoji-a21
   private countryCodeToFlagEmoji(countryCode: String): String {
@@ -61,10 +72,6 @@ export class SelectCrawlerProxy extends LiteElement {
   }
 
   render() {
-    /*if (this.crawlerProxys && this.crawlerProxys.length < 2) {
-      return html``;
-    }*/
-
     return html`
       <sl-select
         name="proxyId"
@@ -75,15 +82,12 @@ export class SelectCrawlerProxy extends LiteElement {
           : msg("No Proxy")}
         hoist
         clearable
+        size=${ifDefined(this.size)}
         @sl-change=${this.onChange}
-        @sl-focus=${() => {
-          // Refetch to keep list up to date
-          void this.fetchOrgProxies();
-        }}
         @sl-hide=${this.stopProp}
         @sl-after-hide=${this.stopProp}
       >
-        ${this.allProxies?.map(
+        ${this.proxyServers.map(
           (server) =>
             html` <sl-option value=${server.id}>
               ${server.country_code
@@ -121,7 +125,7 @@ export class SelectCrawlerProxy extends LiteElement {
   private onChange(e: Event) {
     this.stopProp(e);
 
-    this.selectedProxy = this.allProxies?.find(
+    this.selectedProxy = this.proxyServers.find(
       ({ id }) => id === (e.target as SlSelect).value,
     );
 
@@ -130,7 +134,7 @@ export class SelectCrawlerProxy extends LiteElement {
     }
 
     this.dispatchEvent(
-      new CustomEvent<SelectCrawlerProxyChangeDetail>("on-change", {
+      new CustomEvent<SelectCrawlerProxyChangeDetail>("btrix-change", {
         detail: {
           value: this.selectedProxy ? this.selectedProxy.id : null,
         },
@@ -138,63 +142,24 @@ export class SelectCrawlerProxy extends LiteElement {
     );
   }
 
-  /**
-   * Fetch crawler proxies and update internal state
-   */
-  private async fetchOrgProxies(): Promise<void> {
-    try {
-      const data = await this.getOrgProxies();
-      const defaultProxyId = data.default_proxy_id;
+  private async initProxies(): Promise<void> {
+    const defaultProxyId = this.defaultProxyId;
 
-      this.allProxies = data.servers;
-
-      if (!this.defaultProxy) {
-        this.defaultProxy = this.allProxies.find(
-          ({ id }) => id === defaultProxyId,
-        );
-      }
-
-      if (this.proxyId && !this.selectedProxy?.id) {
-        this.selectedProxy = this.allProxies.find(
-          ({ id }) => id === this.proxyId,
-        );
-      }
-
-      if (!this.selectedProxy) {
-        this.proxyId = null;
-        this.dispatchEvent(
-          new CustomEvent("on-change", {
-            detail: {
-              value: null,
-            },
-          }),
-        );
-        this.selectedProxy = this.allProxies.find(
-          ({ id }) => id === this.proxyId,
-        );
-      }
-
-      this.dispatchEvent(
-        new CustomEvent<SelectCrawlerProxyUpdateDetail>("on-update", {
-          detail: {
-            show: this.allProxies.length > 1,
-          },
-        }),
+    if (!this.defaultProxy) {
+      this.defaultProxy = this.proxyServers.find(
+        ({ id }) => id === defaultProxyId,
       );
-    } catch (e) {
-      this.notify({
-        message: msg("Sorry, couldn't retrieve proxies at this time."),
-        variant: "danger",
-        icon: "exclamation-octagon",
-        id: "proxy-retrieve-status",
-      });
     }
-  }
 
-  private async getOrgProxies(): Promise<ProxiesAPIResponse> {
-    return this.apiFetch<ProxiesAPIResponse>(
-      `/orgs/${this.orgId}/crawlconfigs/crawler-proxies`,
-    );
+    if (this.proxyId && !this.selectedProxy) {
+      this.selectedProxy = this.proxyServers.find(
+        ({ id }) => id === this.proxyId,
+      );
+    }
+
+    if (!this.selectedProxy) {
+      this.proxyId = null;
+    }
   }
 
   /**

frontend/src/context/org.ts

@@ -0,0 +1,7 @@
+import { createContext } from "@lit/context";
+
+import type { ProxiesAPIResponse } from "@/types/crawler";
+
+export type ProxiesContext = ProxiesAPIResponse | null;
+
+export const proxiesContext = createContext<ProxiesContext>("proxies");

frontend/src/features/browser-profiles/new-browser-profile-dialog.ts

@@ -1,5 +1,7 @@
+import { consume } from "@lit/context";
 import { localized, msg, str } from "@lit/localize";
 import { type SlInput } from "@shoelace-style/shoelace";
+import { nothing } from "lit";
 import {
   customElement,
   property,
@@ -7,16 +9,21 @@ import {
   queryAsync,
   state,
 } from "lit/decorators.js";
+import { ifDefined } from "lit/directives/if-defined.js";
 import queryString from "query-string";
 
 import type { Dialog } from "@/components/ui/dialog";
 import { type SelectCrawlerChangeEvent } from "@/components/ui/select-crawler";
 import { type SelectCrawlerProxyChangeEvent } from "@/components/ui/select-crawler-proxy";
+import { proxiesContext, type ProxiesContext } from "@/context/org";
 import LiteElement, { html } from "@/utils/LiteElement";
 
 @localized()
 @customElement("btrix-new-browser-profile-dialog")
 export class NewBrowserProfileDialog extends LiteElement {
+  @consume({ context: proxiesContext, subscribe: true })
+  private readonly proxies?: ProxiesContext;
+
   @property({ type: Boolean })
   open = false;
 
@@ -83,14 +90,22 @@ export class NewBrowserProfileDialog extends LiteElement {
               (this.crawlerChannel = e.detail.value!)}
           ></btrix-select-crawler>
         </div>
-        <div class="mt-4">
-          <btrix-select-crawler-proxy
-            orgId=${this.orgId}
-            .proxyId="${this.proxyId || ""}"
-            @on-change=${(e: SelectCrawlerProxyChangeEvent) =>
-              (this.proxyId = e.detail.value!)}
-          ></btrix-select-crawler-proxy>
-        </div>
+        ${this.proxies?.servers.length
+          ? html`
+              <div class="mt-4">
+                <btrix-select-crawler-proxy
+                  defaultProxyId=${ifDefined(
+                    this.proxies.default_proxy_id ?? undefined,
+                  )}
+                  .proxyServers=${this.proxies.servers}
+                  .proxyId="${this.proxyId || ""}"
+                  @btrix-change=${(e: SelectCrawlerProxyChangeEvent) =>
+                    (this.proxyId = e.detail.value)}
+                ></btrix-select-crawler-proxy>
+              </div>
+            `
+          : nothing}
+
         <input class="invisible size-0" type="submit" />
       </form>
       <div slot="footer" class="flex justify-between">

frontend/src/features/crawl-workflows/workflow-editor.ts

@@ -1,3 +1,4 @@
+import { consume } from "@lit/context";
 import { localized, msg, str } from "@lit/localize";
 import type {
   SlChangeEvent,
@@ -43,6 +44,7 @@ import type { SelectCrawlerProxyChangeEvent } from "@/components/ui/select-crawl
 import type { Tab } from "@/components/ui/tab-list";
 import type { TagInputEvent, TagsChangeEvent } from "@/components/ui/tag-input";
 import type { TimeInputChangeEvent } from "@/components/ui/time-input";
+import { proxiesContext, type ProxiesContext } from "@/context/org";
 import { type SelectBrowserProfileChangeEvent } from "@/features/browser-profiles/select-browser-profile";
 import type { CollectionsChangeEvent } from "@/features/collections/collections-add";
 import type { QueueExclusionTable } from "@/features/crawl-workflows/queue-exclusion-table";
@@ -188,6 +190,9 @@ type CrawlConfigResponse = {
 @localized()
 @customElement("btrix-workflow-editor")
 export class WorkflowEditor extends BtrixElement {
+  @consume({ context: proxiesContext, subscribe: true })
+  private readonly proxies?: ProxiesContext;
+
   @property({ type: String })
   configId?: string;
 
@@ -1329,17 +1334,24 @@ https://archiveweb.page/images/${"logo.svg"}`}
         ></btrix-select-browser-profile>
       `)}
       ${this.renderHelpTextCol(infoTextStrings["browserProfile"])}
-      ${inputCol(html`
-        <btrix-select-crawler-proxy
-          orgId=${this.orgId}
-          .proxyId="${this.formState.proxyId || ""}"
-          @on-change=${(e: SelectCrawlerProxyChangeEvent) =>
-            this.updateFormState({
-              proxyId: e.detail.value,
-            })}
-        ></btrix-select-crawler-proxy>
-      `)}
-      ${this.renderHelpTextCol(infoTextStrings["proxyId"])}
+      ${this.proxies?.servers.length
+        ? [
+            inputCol(html`
+              <btrix-select-crawler-proxy
+                defaultProxyId=${ifDefined(
+                  this.proxies.default_proxy_id ?? undefined,
+                )}
+                .proxyServers=${this.proxies.servers}
+                .proxyId="${this.formState.proxyId || ""}"
+                @btrix-change=${(e: SelectCrawlerProxyChangeEvent) =>
+                  this.updateFormState({
+                    proxyId: e.detail.value,
+                  })}
+              ></btrix-select-crawler-proxy>
+            `),
+            this.renderHelpTextCol(infoTextStrings["proxyId"]),
+          ]
+        : nothing}
       ${inputCol(html`
         <sl-radio-group
           name="scale"

frontend/src/pages/org/index.ts

@@ -1,3 +1,4 @@
+import { provide } from "@lit/context";
 import { localized, msg, str } from "@lit/localize";
 import { html, nothing } from "lit";
 import { customElement, property, state } from "lit/decorators.js";
@@ -15,10 +16,12 @@ import type {
 } from "./settings/settings";
 
 import { BtrixElement } from "@/classes/BtrixElement";
+import { proxiesContext, type ProxiesContext } from "@/context/org";
 import type { QuotaUpdateDetail } from "@/controllers/api";
 import needLogin from "@/decorators/needLogin";
 import type { CollectionSavedEvent } from "@/features/collections/collection-metadata-dialog";
 import type { SelectJobTypeEvent } from "@/features/crawl-workflows/new-workflow-dialog";
+import type { ProxiesAPIResponse } from "@/types/crawler";
 import type { UserOrg } from "@/types/user";
 import { isApiError } from "@/utils/api";
 import type { ViewState } from "@/utils/APIRouter";
@@ -93,6 +96,9 @@ const UUID_REGEX =
 @customElement("btrix-org")
 @needLogin
 export class Org extends BtrixElement {
+  @provide({ context: proxiesContext })
+  proxies: ProxiesContext = null;
+
   @property({ type: Object })
   viewStateData?: ViewState["data"];
 
@@ -147,6 +153,7 @@ export class Org extends BtrixElement {
     ) {
       if (this.userOrg) {
         void this.updateOrg();
+        void this.updateOrgProxies();
       } else {
         // Couldn't find org with slug, redirect to first org
         const org = this.userInfo.orgs[0] as UserOrg | undefined;
@@ -211,6 +218,14 @@ export class Org extends BtrixElement {
     }
   }
 
+  private async updateOrgProxies() {
+    try {
+      this.proxies = await this.getOrgProxies(this.orgId);
+    } catch (e) {
+      console.debug(e);
+    }
+  }
+
   async firstUpdated() {
     // if slug is actually an orgId (UUID), attempt to lookup the slug
     // and redirect to the slug url
@@ -229,6 +244,8 @@ export class Org extends BtrixElement {
     // Sync URL to create dialog
     const dialogName = this.getDialogName();
     if (dialogName) this.openDialog(dialogName);
+
+    void this.updateOrgProxies();
   }
 
   private getDialogName() {
@@ -634,6 +651,12 @@ export class Org extends BtrixElement {
     return data;
   }
 
+  private async getOrgProxies(orgId: string): Promise<ProxiesAPIResponse> {
+    return this.api.fetch<ProxiesAPIResponse>(
+      `/orgs/${orgId}/crawlconfigs/crawler-proxies`,
+    );
+  }
+
   private async onOrgRemoveMember(e: OrgRemoveMemberEvent) {
     void this.removeMember(e.detail.member);
   }
@@ -682,9 +705,15 @@ export class Org extends BtrixElement {
         icon: "check2-circle",
         id: "user-updated-status",
       });
+
       const org = await this.getOrg(this.orgId);
 
-      AppStateService.updateOrg(org);
+      if (org) {
+        AppStateService.partialUpdateOrg({
+          id: org.id,
+          users: org.users,
+        });
+      }
     } catch (e) {
       console.debug(e);
 
@@ -742,7 +771,12 @@ export class Org extends BtrixElement {
       } else {
         const org = await this.getOrg(this.orgId);
 
-        AppStateService.updateOrg(org);
+        if (org) {
+          AppStateService.partialUpdateOrg({
+            id: org.id,
+            users: org.users,
+          });
+        }
       }
     } catch (e) {
       console.debug(e);

frontend/src/pages/org/settings/components/crawling-defaults.ts

@@ -1,3 +1,4 @@
+import { consume } from "@lit/context";
 import { localized, msg } from "@lit/localize";
 import type { SlButton } from "@shoelace-style/shoelace";
 import { serialize } from "@shoelace-style/shoelace/dist/utilities/form.js";
@@ -9,6 +10,8 @@ import { ifDefined } from "lit/directives/if-defined.js";
 
 import { BtrixElement } from "@/classes/BtrixElement";
 import type { LanguageSelect } from "@/components/ui/language-select";
+import type { SelectCrawlerProxy } from "@/components/ui/select-crawler-proxy";
+import { proxiesContext, type ProxiesContext } from "@/context/org";
 import type { QueueExclusionTable } from "@/features/crawl-workflows/queue-exclusion-table";
 import { columns, type Cols } from "@/layouts/columns";
 import infoTextStrings from "@/strings/crawl-workflows/infoText";
@@ -25,7 +28,7 @@ import {
 } from "@/utils/workflow";
 
 type FieldName = keyof FormState;
-type Field = Record<FieldName, TemplateResult<1>>;
+type Field = Record<FieldName, TemplateResult<1> | undefined>;
 
 const PLACEHOLDER_EXCLUSIONS = [""]; // Add empty slot
 
@@ -51,6 +54,9 @@ export class OrgSettingsCrawlWorkflows extends BtrixElement {
     }
   `;
 
+  @consume({ context: proxiesContext, subscribe: true })
+  private readonly proxies?: ProxiesContext;
+
   @state()
   private defaults: WorkflowDefaults = appDefaults;
 
@@ -60,6 +66,9 @@ export class OrgSettingsCrawlWorkflows extends BtrixElement {
   @query("btrix-language-select")
   languageSelect?: LanguageSelect | null;
 
+  @query("btrix-select-crawler-proxy")
+  proxySelect?: SelectCrawlerProxy | null;
+
   @query('sl-button[type="submit"]')
   submitButton?: SlButton | null;
 
@@ -195,10 +204,16 @@ export class OrgSettingsCrawlWorkflows extends BtrixElement {
           size="small"
         ></btrix-select-browser-profile>
       `,
-      proxyId: html` <btrix-select-crawler-proxy
-        orgId=${this.orgId}
-        .proxyId="${orgDefaults.proxyId || null}"
-      ></btrix-select-crawler-proxy>`,
+      proxyId: this.proxies?.servers.length
+        ? html` <btrix-select-crawler-proxy
+            defaultProxyId=${ifDefined(
+              this.proxies.default_proxy_id ?? undefined,
+            )}
+            .proxyServers=${this.proxies.servers}
+            .proxyId="${orgDefaults.proxyId || null}"
+            size="small"
+          ></btrix-select-crawler-proxy>`
+        : undefined,
       crawlerChannel: html`
         <btrix-select-crawler
           crawlerChannel=${ifDefined(orgDefaults.crawlerChannel)}
@@ -254,10 +269,12 @@ export class OrgSettingsCrawlWorkflows extends BtrixElement {
             Object.entries(this.fields).map(([sectionName, fields]) =>
               section(
                 sectionName as SectionsEnum,
-                Object.entries(fields).map(([fieldName, field]) => [
-                  field,
-                  infoTextStrings[fieldName as FieldName],
-                ]),
+                Object.entries(fields)
+                  .filter(([, field]) => field as unknown)
+                  .map(([fieldName, field]) => [
+                    field,
+                    infoTextStrings[fieldName as FieldName],
+                  ]),
               ),
             ),
           )}
@@ -291,7 +308,7 @@ export class OrgSettingsCrawlWorkflows extends BtrixElement {
       blockAds: values.blockAds === "on",
       profileid: values.profileid,
       crawlerChannel: values.crawlerChannel,
-      proxyId: values.proxyId,
+      proxyId: this.proxySelect?.value || undefined,
       userAgent: values.userAgent,
       lang: this.languageSelect?.value || undefined,
       exclude: this.exclusionTable?.exclusions?.filter((v) => v) || [],

frontend/yarn.lock

@@ -1161,6 +1161,13 @@
   resolved "https://registry.yarnpkg.com/@lit-labs/ssr-dom-shim/-/ssr-dom-shim-1.2.0.tgz#353ce4a76c83fadec272ea5674ede767650762fd"
   integrity sha512-yWJKmpGE6lUURKAaIltoPIE/wrbY3TEkqQt+X0m+7fQNnAv0keydnYvbiJFP1PnMhizmIWRWOG5KLhYyc/xl+g==
 
+"@lit/context@^1.1.3":
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/@lit/context/-/context-1.1.3.tgz#66f8832e57f760f51f39c9d658ca6bd78f809e19"
+  integrity sha512-Auh37F4S0PZM93HTDfZWs97mmzaQ7M3vnTc9YvxAGyP3UItSK/8Fs0vTOGT+njuvOwbKio/l8Cx/zWL4vkutpQ==
+  dependencies:
+    "@lit/reactive-element" "^1.6.2 || ^2.0.0"
+
 "@lit/localize-tools@^0.8.0":
   version "0.8.0"
   resolved "https://registry.yarnpkg.com/@lit/localize-tools/-/localize-tools-0.8.0.tgz#8a14b3961aa63ef801c9f63274cb1d58821a3552"
@@ -1197,7 +1204,7 @@
   dependencies:
     "@lit-labs/ssr-dom-shim" "^1.0.0"
 
-"@lit/reactive-element@^1.0.0 || ^2.0.0":
+"@lit/reactive-element@^1.0.0 || ^2.0.0", "@lit/reactive-element@^1.6.2 || ^2.0.0":
   version "2.0.4"
   resolved "https://registry.yarnpkg.com/@lit/reactive-element/-/reactive-element-2.0.4.tgz#8f2ed950a848016383894a26180ff06c56ae001b"
   integrity sha512-GFn91inaUa2oHLak8awSIigYz0cU0Payr1rcFsrkf5OJ5eSPxElyZfKh0f2p9FsTiZWXQdWGJeXZICEfXXYSXQ==