browsertrix/frontend/src/controllers/api.ts
Ilya Kreymer 795a1a6f58
feat: Frontend upload seed url list (#2761)
Resolves #2646
Depends on #2710

## Changes

(Copied from #2689)

- Allows users to specify URL list as file.
- Allow uploading a text file of URLs
- Allow specifying >100 URLs into URL list, where they will turn into an uploaded list automatically.


---------
Co-authored-by: sua yoo <sua@suayoo.com>
2025-07-22 20:17:27 -07:00

265 lines
7.0 KiB
TypeScript

import { msg } from "@lit/localize";
import type { ReactiveController, ReactiveControllerHost } from "lit";
import throttle from "lodash/fp/throttle";
import { APIError } from "@/utils/api";
import AuthService from "@/utils/AuthService";
import appState from "@/utils/state";
export type QuotaUpdateDetail = { reached: boolean };
export interface APIEventMap {
"btrix-execution-minutes-quota-update": CustomEvent<QuotaUpdateDetail>;
"btrix-storage-quota-update": CustomEvent<QuotaUpdateDetail>;
}
export enum AbortReason {
UserCancel = "user-canceled",
QuotaReached = "storage_quota_reached",
NetworkError = "network-error",
RequestTimeout = "request-timeout",
}
/**
* Utilities for interacting with the Browsertrix backend API
*
* @example Usage:
* ```ts
* class MyComponent extends LitElement {
* private api = new APIController(this);
*
* async getSomething() {
* await this.api.fetch("/path")
* }
* }
* ```
*/
export class APIController implements ReactiveController {
host: ReactiveControllerHost & EventTarget;
uploadProgress = 0;
private uploadRequest: XMLHttpRequest | null = null;
constructor(host: APIController["host"]) {
this.host = host;
host.addController(this);
}
hostConnected() {}
hostDisconnected() {
this.cancelUpload();
}
async fetch<T = unknown>(path: string, options?: RequestInit): Promise<T> {
const auth = appState.auth;
if (!auth) throw new Error("auth not in state");
const { headers, ...opts } = options || {};
const resp = await fetch("/api" + path, {
headers: {
"Content-Type": "application/json",
...headers,
...auth.headers,
},
...opts,
});
if (resp.ok) {
const body = await resp.json();
const storageQuotaReached = body.storageQuotaReached;
const executionMinutesQuotaReached = body.execMinutesQuotaReached;
if (typeof storageQuotaReached === "boolean") {
if (storageQuotaReached !== appState.org?.storageQuotaReached) {
this.host.dispatchEvent(
new CustomEvent<QuotaUpdateDetail>("btrix-storage-quota-update", {
detail: { reached: storageQuotaReached },
bubbles: true,
composed: true,
}),
);
}
}
if (typeof executionMinutesQuotaReached === "boolean") {
if (
executionMinutesQuotaReached != appState.org?.execMinutesQuotaReached
) {
this.host.dispatchEvent(
new CustomEvent<QuotaUpdateDetail>(
"btrix-execution-minutes-quota-update",
{
detail: { reached: executionMinutesQuotaReached },
bubbles: true,
composed: true,
},
),
);
}
}
return body as T;
}
let errorDetail;
let errorDetails = null;
try {
errorDetail = (await resp.json()).detail;
} catch {
/* empty */
}
let errorMessage: string = msg("Unknown API error");
switch (resp.status) {
case 401: {
this.host.dispatchEvent(AuthService.createNeedLoginEvent());
errorMessage = msg("Need login");
break;
}
case 403: {
if (errorDetail === "storage_quota_reached") {
this.host.dispatchEvent(
new CustomEvent<QuotaUpdateDetail>("btrix-storage-quota-update", {
detail: { reached: true },
bubbles: true,
composed: true,
}),
);
errorMessage = msg("Storage quota reached");
break;
}
if (errorDetail === "exec_minutes_quota_reached") {
this.host.dispatchEvent(
new CustomEvent<QuotaUpdateDetail>(
"btrix-execution-minutes-quota-update",
{
detail: { reached: true },
bubbles: true,
composed: true,
},
),
);
errorMessage = msg("Monthly execution minutes quota reached");
break;
}
break;
}
case 404: {
errorMessage = msg("Not found");
break;
}
default: {
if (typeof errorDetail === "string") {
errorMessage = errorDetail;
errorDetails = [errorDetail];
} else if (Array.isArray(errorDetail) && errorDetail.length) {
errorDetails = errorDetail;
const fieldDetail = errorDetail[0] || {};
const { loc, msg } = fieldDetail;
const fieldName = loc
.filter((v: unknown) => v !== "body" && typeof v === "string")
.join(" ");
errorMessage = `${fieldName} ${msg}`;
}
break;
}
}
throw new APIError({
message: errorMessage,
status: resp.status,
details: errorDetails,
errorCode: errorDetail,
});
}
async upload(
path: string,
file: File,
abortSignal?: AbortSignal,
): Promise<{ id: string; added: boolean; storageQuotaReached: boolean }> {
const auth = appState.auth;
if (!auth) throw new Error("auth not in state");
// TODO handle multiple uploads
if (this.uploadRequest) {
console.debug("upload request exists");
this.cancelUpload();
}
return new Promise((resolve, reject) => {
if (abortSignal?.aborted) {
reject(AbortReason.UserCancel);
}
const xhr = new XMLHttpRequest();
xhr.open("PUT", `/api${path}`);
xhr.setRequestHeader("Content-Type", "application/octet-stream");
Object.entries(auth.headers).forEach(([k, v]) => {
xhr.setRequestHeader(k, v);
});
xhr.addEventListener("load", () => {
if (xhr.status === 200) {
resolve(
JSON.parse(xhr.response as string) as {
id: string;
added: boolean;
storageQuotaReached: boolean;
},
);
}
if (xhr.status === 403) {
reject(AbortReason.QuotaReached);
}
if (xhr.status >= 404) {
reject(
new APIError({
message: xhr.statusText,
status: xhr.status,
}),
);
}
});
xhr.addEventListener("error", () => {
reject(AbortReason.NetworkError);
});
xhr.addEventListener("timeout", () => {
reject(AbortReason.RequestTimeout);
});
xhr.addEventListener("abort", () => {
reject(AbortReason.UserCancel);
});
xhr.upload.addEventListener("progress", this.onUploadProgress);
xhr.send(file);
abortSignal?.addEventListener("abort", () => {
xhr.abort();
reject(AbortReason.UserCancel);
});
this.uploadRequest = xhr;
});
}
readonly onUploadProgress = throttle(100)((e: ProgressEvent) => {
this.uploadProgress = (e.loaded / e.total) * 100;
this.host.requestUpdate();
});
private cancelUpload() {
if (this.uploadRequest) {
this.uploadRequest.abort();
this.uploadRequest = null;
}
this.onUploadProgress.cancel();
}
}