1.12.2 release -> main (#2181)

Merge 1.12.2 release changes into main, includes:
- Collection replay full refresh on metadata / archived items (#2176)
- Fix for self-registration default org (#2178)
- Prepend missing https in start URL (#2177)
- Updated billing to support free trial messaging (#2179)

---------

Co-authored-by: sua yoo <sua@webrecorder.org>
Co-authored-by: Henry Wilkinson <henry@wilkinson.graphics>
Co-authored-by: sua yoo <sua@suayoo.com>
Co-authored-by: SuaYoo <SuaYoo@users.noreply.github.com>
This commit is contained in:
Ilya Kreymer 2024-11-26 11:17:07 -08:00 committed by GitHub
parent 37c0b06622
commit 50dac7dc50
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 242 additions and 127 deletions

View File

@ -115,6 +115,7 @@ class OrgOps:
invites: InviteOps
user_manager: UserManager
register_to_org_id: Optional[str]
base_crawl_ops: BaseCrawlOps
default_primary: Optional[StorageRef]
@ -295,7 +296,7 @@ class OrgOps:
"""Get default organiation for new user registration, or default org"""
if self.register_to_org_id:
try:
await self.get_org_by_id(UUID(self.register_to_org_id))
return await self.get_org_by_id(UUID(self.register_to_org_id))
except HTTPException as exc:
raise HTTPException(
status_code=500, detail="default_register_org_not_found"

View File

@ -143,7 +143,7 @@ class StorageOps:
use_access_for_presign = False
else:
access_endpoint_url = storage.get("access_endpoint_url") or endpoint_url
use_access_for_presign = True
use_access_for_presign = is_bool(storage.get("use_access_for_presign"))
return S3Storage(
access_key=storage["access_key"],

View File

@ -75,7 +75,7 @@ allow_dupe_invites: "0"
invite_expire_seconds: 604800
# base url for replayweb.page
rwp_base_url: "https://cdn.jsdelivr.net/npm/replaywebpage@2.1.4/"
rwp_base_url: "https://cdn.jsdelivr.net/npm/replaywebpage@2.2.4/"
superuser:
# set this to enable a superuser admin

View File

@ -555,7 +555,7 @@ export class CollectionItemsDialog extends BtrixElement {
let selectionMessage = msg("No changes to save");
if (hasChange) {
const messages = [];
const messages: string[] = [];
if (addCount) {
messages.push(
msg(
@ -565,7 +565,9 @@ export class CollectionItemsDialog extends BtrixElement {
}
if (removeCount) {
messages.push(
str`Adding ${this.localize.number(removeCount)} ${pluralOf("items", removeCount)}`,
msg(
str`Removing ${this.localize.number(removeCount)} ${pluralOf("items", removeCount)}`,
),
);
}

View File

@ -112,7 +112,6 @@ const DEFAULT_BEHAVIORS = [
"autofetch",
"siteSpecific",
];
const MAX_ADDITIONAL_URLS = 100;
const getDefaultProgressState = (hasConfigId = false): ProgressState => {
let activeTab: StepName = "crawlSetup";
@ -163,7 +162,8 @@ function getLocalizedWeekDays() {
}
function validURL(url: string) {
return /((((https?):(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/\\\w]*))?)/.test(
// adapted from: https://gist.github.com/dperini/729294
return /^(?:https?:\/\/)?(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z0-9\u00a1-\uffff][a-z0-9\u00a1-\uffff_-]{0,62})?[a-z0-9\u00a1-\uffff]\.)+(?:[a-z\u00a1-\uffff]{2,}\.?))(?::\d{2,5})?(?:[/?#]\S*)?$/i.test(
url,
);
}
@ -174,7 +174,8 @@ const urlListToArray = flow(
trimArray,
);
const URL_LIST_MAX_URLS = 1000;
//todo: make this customizable, perhaps at deploy time
const URL_LIST_MAX_URLS = 100;
type CrawlConfigResponse = {
run_now_job?: boolean;
@ -814,6 +815,17 @@ export class WorkflowEditor extends BtrixElement {
const text = msg("Please enter a valid URL.");
inputEl.helpText = text;
inputEl.setCustomValidity(text);
} else if (
inputEl.value &&
!inputEl.value.startsWith("https://") &&
!inputEl.value.startsWith("http://")
) {
this.updateFormState(
{
urlList: "https://" + inputEl.value,
},
true,
);
}
}}
>
@ -835,19 +847,8 @@ https://archiveweb.page/guide`}
required
@keyup=${async (e: KeyboardEvent) => {
if (e.key === "Enter") {
const inputEl = e.target as SlInput;
await inputEl.updateComplete;
if (!inputEl.value) return;
const { isValid, helpText } = this.validateUrlList(
inputEl.value,
MAX_ADDITIONAL_URLS,
);
inputEl.helpText = helpText;
if (isValid) {
inputEl.setCustomValidity("");
} else {
inputEl.setCustomValidity(helpText);
}
await (e.target as SlInput).updateComplete;
this.doValidateTextArea(e.target);
}
}}
@sl-input=${(e: CustomEvent) => {
@ -857,24 +858,16 @@ https://archiveweb.page/guide`}
}
}}
@sl-change=${async (e: CustomEvent) => {
const inputEl = e.target as SlInput;
if (!inputEl.value) return;
const { isValid, helpText } = this.validateUrlList(
inputEl.value,
MAX_ADDITIONAL_URLS,
);
inputEl.helpText = helpText;
if (isValid) {
inputEl.setCustomValidity("");
} else {
inputEl.setCustomValidity(helpText);
}
this.doValidateTextArea(e.target);
}}
@sl-blur=${async (e: CustomEvent) => {
this.doValidateTextArea(e.target);
}}
></sl-textarea>
`)}
${this.renderHelpTextCol(
msg(
str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(MAX_ADDITIONAL_URLS)} URLs.`,
str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(URL_LIST_MAX_URLS)} URLs.`,
),
)}
`}
@ -997,6 +990,17 @@ https://archiveweb.page/guide`}
const text = msg("Please enter a valid URL.");
inputEl.helpText = text;
inputEl.setCustomValidity(text);
} else if (
inputEl.value &&
!inputEl.value.startsWith("https://") &&
!inputEl.value.startsWith("http://")
) {
this.updateFormState(
{
primarySeedUrl: "https://" + inputEl.value,
},
true,
);
}
}}
>
@ -1099,19 +1103,8 @@ https://example.net`}
https://archiveweb.page/images/${"logo.svg"}`}
@keyup=${async (e: KeyboardEvent) => {
if (e.key === "Enter") {
const inputEl = e.target as SlInput;
await inputEl.updateComplete;
if (!inputEl.value) return;
const { isValid, helpText } = this.validateUrlList(
inputEl.value,
MAX_ADDITIONAL_URLS,
);
inputEl.helpText = helpText;
if (isValid) {
inputEl.setCustomValidity("");
} else {
inputEl.setCustomValidity(helpText);
}
await (e.target as SlInput).updateComplete;
this.doValidateTextArea(e.target);
}
}}
@sl-input=${(e: CustomEvent) => {
@ -1121,24 +1114,16 @@ https://archiveweb.page/images/${"logo.svg"}`}
}
}}
@sl-change=${async (e: CustomEvent) => {
const inputEl = e.target as SlInput;
if (!inputEl.value) return;
const { isValid, helpText } = this.validateUrlList(
inputEl.value,
MAX_ADDITIONAL_URLS,
);
inputEl.helpText = helpText;
if (isValid) {
inputEl.setCustomValidity("");
} else {
inputEl.setCustomValidity(helpText);
}
this.doValidateTextArea(e.target);
}}
@sl-blur=${async (e: CustomEvent) => {
this.doValidateTextArea(e.target);
}}
></sl-textarea>
`)}
${this.renderHelpTextCol(
msg(
str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(MAX_ADDITIONAL_URLS)} URLs.`,
str`The crawler will visit and record each URL listed here. You can enter up to ${this.localize.number(URL_LIST_MAX_URLS)} URLs.`,
),
)}
</div>
@ -1147,6 +1132,21 @@ https://archiveweb.page/images/${"logo.svg"}`}
`;
};
private doValidateTextArea(target: EventTarget | null) {
const inputEl = target as SlInput;
if (!inputEl.value) return;
const { isValid, helpText } = this.validateUrlList(
inputEl.value,
URL_LIST_MAX_URLS,
);
inputEl.helpText = helpText;
if (isValid) {
inputEl.setCustomValidity("");
} else {
inputEl.setCustomValidity(helpText);
}
}
private renderCrawlLimits() {
// Max Pages minimum value cannot be lower than seed count
const minPages = Math.max(
@ -2076,6 +2076,20 @@ https://archiveweb.page/images/${"logo.svg"}`}
str`Please remove or fix the following invalid URL: ${invalidUrl}`,
);
}
if (isValid) {
// auto-add https:// prefix if otherwise a valid URL
let updated = false;
for (let i = 0; i < urlList.length; i++) {
const url = urlList[i];
if (!url.startsWith("http://") && !url.startsWith("https://")) {
urlList[i] = "https://" + url;
updated = true;
}
}
if (updated) {
this.updateFormState({ urlList: urlList.join("\n") });
}
}
}
return { isValid, helpText };
}

View File

@ -4,6 +4,7 @@ import { html, type TemplateResult } from "lit";
import { customElement } from "lit/decorators.js";
import { BtrixElement } from "@/classes/BtrixElement";
import { SubscriptionStatus } from "@/types/billing";
import { OrgReadOnlyReason } from "@/types/org";
type Alert = {
@ -61,16 +62,32 @@ export class OrgStatusBanner extends BtrixElement {
execMinutesQuotaReached,
} = this.org;
let daysDiff = 0;
let dateStr = "";
const futureCancelDate = subscription?.futureCancelDate || null;
if (futureCancelDate) {
daysDiff = differenceInDays(new Date(), new Date(futureCancelDate));
dateStr = this.localize.date(futureCancelDate, {
month: "long",
day: "numeric",
year: "numeric",
hour: "numeric",
});
}
const isTrial = subscription?.status === SubscriptionStatus.Trialing;
// show banner if < this many days of trial is left
const MAX_TRIAL_DAYS_SHOW_BANNER = 4;
return [
{
test: () =>
!readOnly && !readOnlyOnCancel && !!subscription?.futureCancelDate,
!readOnly && !readOnlyOnCancel && !!futureCancelDate && !isTrial,
content: () => {
const daysDiff = differenceInDays(
new Date(),
new Date(subscription!.futureCancelDate!),
);
return {
title:
daysDiff > 1
@ -82,15 +99,7 @@ export class OrgStatusBanner extends BtrixElement {
detail: html`
<p>
${msg(
str`Your subscription ends on ${this.localize.date(
subscription!.futureCancelDate!,
{
month: "long",
day: "numeric",
year: "numeric",
hour: "numeric",
},
)}. Your user account, org, and all associated data will be deleted.`,
str`Your subscription ends on ${dateStr}. Your user account, org, and all associated data will be deleted.`,
)}
</p>
<p>
@ -106,13 +115,43 @@ export class OrgStatusBanner extends BtrixElement {
},
{
test: () =>
!readOnly && readOnlyOnCancel && !!subscription?.futureCancelDate,
!readOnly &&
!readOnlyOnCancel &&
!!futureCancelDate &&
isTrial &&
daysDiff < MAX_TRIAL_DAYS_SHOW_BANNER,
content: () => {
return {
title:
daysDiff > 1
? msg(
str`You have ${daysDiff} days left of your Browsertrix trial`,
)
: msg(`Your trial ends within one day`),
detail: html`
<p>
${msg(
html`Your free trial ends on ${dateStr}. To continue using
Browsertrix, select <strong>Choose Plan</strong> in
${billingTabLink}.`,
)}
</p>
<p>
${msg(
str`Your web archives are always yours — you can download any archived items you'd like to keep
before the trial ends!`,
)}
</p>
`,
};
},
},
{
test: () => !readOnly && readOnlyOnCancel && !!futureCancelDate,
content: () => {
const daysDiff = differenceInDays(
new Date(),
new Date(subscription!.futureCancelDate!),
);
return {
title:
daysDiff > 1
@ -121,20 +160,12 @@ export class OrgStatusBanner extends BtrixElement {
detail: html`
<p>
${msg(
str`Your subscription ends on ${this.localize.date(
subscription!.futureCancelDate!,
{
month: "long",
day: "numeric",
year: "numeric",
hour: "numeric",
},
)}. You will no longer be able to run crawls, upload files, create browser profiles, or create collections.`,
str`Your subscription ends on ${dateStr}. You will no longer be able to run crawls, upload files, create browser profiles, or create collections.`,
)}
</p>
<p>
${msg(
html`To keep your plan and continue crawling, see
html`To choose a plan and continue using Browsertrix, see
${billingTabLink}.`,
)}
</p>

View File

@ -60,6 +60,9 @@ export class CollectionDetail extends BtrixElement {
@query(".descriptionExpandBtn")
private readonly descriptionExpandBtn?: HTMLElement | null;
@query("replay-web-page")
private readonly replayEmbed?: ReplayWebPage | null;
// Use to cancel requests
private getArchivedItemsController: AbortController | null = null;
@ -203,6 +206,7 @@ export class CollectionDetail extends BtrixElement {
?open=${this.openDialogName === "editItems"}
@sl-hide=${() => (this.openDialogName = undefined)}
@btrix-collection-saved=${() => {
this.refreshReplay();
void this.fetchCollection();
void this.fetchArchivedItems();
}}
@ -215,7 +219,10 @@ export class CollectionDetail extends BtrixElement {
.collection=${this.collection!}
?open=${this.openDialogName === "editMetadata"}
@sl-hide=${() => (this.openDialogName = undefined)}
@btrix-collection-saved=${() => void this.fetchCollection()}
@btrix-collection-saved=${() => {
this.refreshReplay();
void this.fetchCollection();
}}
>
</btrix-collection-metadata-dialog>
`,
@ -223,6 +230,16 @@ export class CollectionDetail extends BtrixElement {
${this.renderShareDialog()}`;
}
private refreshReplay() {
if (this.replayEmbed) {
try {
this.replayEmbed.fullReload();
} catch (e) {
console.warn("Full reload not available in RWP");
}
}
}
private getPublicReplayURL() {
return new URL(
`/api/orgs/${this.orgId}/collections/${this.collectionId}/public/replay.json`,

View File

@ -18,7 +18,7 @@ import { tw } from "@/utils/tailwind";
const linkClassList = tw`transition-color text-primary hover:text-primary-500`;
const manageLinkClasslist = clsx(
linkClassList,
tw`flex items-center gap-2 p-2 text-sm font-semibold leading-none`,
tw`flex cursor-pointer items-center gap-2 p-2 text-sm font-semibold leading-none`,
);
@localized()
@ -41,6 +41,10 @@ export class OrgSettingsBilling extends BtrixElement {
let label = msg("Manage Billing");
switch (subscription.status) {
case SubscriptionStatus.Trialing: {
label = msg("Choose Plan");
break;
}
case SubscriptionStatus.PausedPaymentFailed: {
label = msg("Update Billing");
break;
@ -112,32 +116,41 @@ export class OrgSettingsBilling extends BtrixElement {
</div>
${when(
this.org,
(org) =>
org.subscription?.futureCancelDate
? html`
<div
class="mb-3 flex items-center gap-2 border-b pb-3 text-neutral-500"
>
<sl-icon
name="info-circle"
class="text-base"
></sl-icon>
<span>
${msg(
(org) => {
if (!org.subscription?.futureCancelDate) {
return nothing;
}
const futureCancelDate = html`<sl-format-date
class="truncate"
date=${org.subscription.futureCancelDate}
month="long"
day="numeric"
year="numeric"
>
</sl-format-date>`;
return html`
<div
class="mb-3 flex items-center gap-2 border-b pb-3 text-neutral-500"
>
<sl-icon name="info-circle" class="text-base"></sl-icon>
<span>
${org.subscription.status ===
SubscriptionStatus.Trialing
? msg(
html`Your trial will end on ${futureCancelDate}
- Click <strong>Choose Plan</strong> to
subscribe`,
)
: msg(
html`Your plan will be canceled on
<sl-format-date
class="truncate"
date=${org.subscription.futureCancelDate}
month="long"
day="numeric"
year="numeric"
>
</sl-format-date>`,
${futureCancelDate}`,
)}
</span>
</div>
`
: nothing,
</span>
</div>
`;
},
() => html` <sl-skeleton></sl-skeleton> `,
)}
<h5 class="mb-2 mt-4 text-xs leading-none text-neutral-500">
@ -245,6 +258,12 @@ export class OrgSettingsBilling extends BtrixElement {
`;
break;
}
case SubscriptionStatus.Trialing: {
statusLabel = html`
<span class="text-success-700">${msg("Trial")}</span>
`;
break;
}
case SubscriptionStatus.PausedPaymentFailed: {
statusLabel = html`
<span class="text-danger">${msg("Paused, payment failed")}</span>

View File

@ -1,4 +1,6 @@
/**
* @TODO Import from replaywebpage once https://github.com/webrecorder/replayweb.page/issues/376 is addressed
*
* @attr {String} source
* @attr {String} coll
* @attr {String} config
@ -7,7 +9,9 @@
* @attr {String} noCache
* @attr {String} url
*/
class ReplayWebPage {}
class ReplayWebPage {
fullReload(): void {}
}
declare global {
interface HTMLElementTagNameMap {

View File

@ -4,6 +4,7 @@ import { apiDateSchema } from "./api";
export enum SubscriptionStatus {
Active = "active",
Trialing = "trialing",
PausedPaymentFailed = "paused_payment_failed",
Cancelled = "cancelled",
}

View File

@ -2947,7 +2947,7 @@
<x equiv-text="${daysDiff}" id="0"/> days</source>
</trans-unit>
<trans-unit id="s7fa0d24b94690373">
<source>Your subscription ends on <x equiv-text="${this.localize.date(subscription!.futureCancelDate!, {&#10; month: &quot;long&quot;,&#10; day: &quot;numeric&quot;,&#10; year: &quot;numeric&quot;,&#10; hour: &quot;numeric&quot;,&#10;})}" id="0"/>. Your user account, org, and all associated data will be deleted.</source>
<source>Your subscription ends on <x equiv-text="${dateStr}" id="0"/>. Your user account, org, and all associated data will be deleted.</source>
</trans-unit>
<trans-unit id="h16be212de6638b6c">
<source>We suggest downloading your archived items before they
@ -2961,11 +2961,7 @@
<source>Archiving will be disabled within one day</source>
</trans-unit>
<trans-unit id="s618b35a93b6fd392">
<source>Your subscription ends on <x equiv-text="${this.localize.date(subscription!.futureCancelDate!, {&#10; month: &quot;long&quot;,&#10; day: &quot;numeric&quot;,&#10; year: &quot;numeric&quot;,&#10; hour: &quot;numeric&quot;,&#10;})}" id="0"/>. You will no longer be able to run crawls, upload files, create browser profiles, or create collections.</source>
</trans-unit>
<trans-unit id="hf17c5369da37401b">
<source>To keep your plan and continue crawling, see
<x equiv-text="${billingTabLink}" id="0"/>.</source>
<source>Your subscription ends on <x equiv-text="${dateStr}" id="0"/>. You will no longer be able to run crawls, upload files, create browser profiles, or create collections.</source>
</trans-unit>
<trans-unit id="sfb85ab2a166e4c99">
<source>Archiving is disabled for this org</source>
@ -3671,7 +3667,7 @@
<source>The URL of the page to crawl.</source>
</trans-unit>
<trans-unit id="s41d2278219615589">
<source>The crawler will visit and record each URL listed here. You can enter up to <x equiv-text="${this.localize.number(MAX_ADDITIONAL_URLS)}" id="0"/> URLs.</source>
<source>The crawler will visit and record each URL listed here. You can enter up to <x equiv-text="${this.localize.number(URL_LIST_MAX_URLS)}" id="0"/> URLs.</source>
</trans-unit>
<trans-unit id="sfc5e402f8b21ef5f">
<source>If checked, the crawler will visit pages one link away.</source>
@ -3787,10 +3783,6 @@
<trans-unit id="seb49ad0f81062f64">
<source>Choose your preferred language for displaying Browsertrix in your browser.</source>
</trans-unit>
<trans-unit id="h746ce875ddd39a65">
<source>Your plan will be canceled on
<x equiv-text="&lt;sl-format-date class=&quot;truncate&quot; date=&quot;${org.subscription.futureCancelDate}&quot; month=&quot;long&quot; day=&quot;numeric&quot; year=&quot;numeric&quot;&gt;&#10; &lt;/sl-format-date&gt;" id="0"/></source>
</trans-unit>
<trans-unit id="h88cfbf4cb1b57616">
<source>Deleting an org will delete all
<x equiv-text="&lt;strong class=&quot;font-semibold&quot;&gt;&#10; &lt;sl-format-bytes value=&quot;${org.bytesStored}&quot;&gt;&lt;/sl-format-bytes&gt;&#10; &lt;/strong&gt;" id="0"/>
@ -3808,6 +3800,40 @@
<source>Profiles:
<x equiv-text="&lt;sl-format-bytes value=&quot;${org.bytesStoredProfiles}&quot;&gt;&lt;/sl-format-bytes&gt;" id="0"/></source>
</trans-unit>
<trans-unit id="se3d7a30d5e45c393">
<source>Trial</source>
</trans-unit>
<trans-unit id="s582e36ff4a424786">
<source>Removing <x equiv-text="${this.localize.number(removeCount)} ${pluralOf(&quot;items&quot;, removeCount)}" id="0"/></source>
</trans-unit>
<trans-unit id="s1f1b3cea8b3a20f3">
<source>You have <x equiv-text="${daysDiff}" id="0"/> days left of your Browsertrix trial</source>
</trans-unit>
<trans-unit id="se4dfda71fd51327d">
<source>Your trial ends within one day</source>
</trans-unit>
<trans-unit id="he8a019fc239da9d2">
<source>Your free trial ends on <x equiv-text="${dateStr}" id="0"/>. To continue using
Browsertrix, select <x equiv-text="&lt;strong&gt;" id="1"/>Choose Plan<x equiv-text="&lt;/strong&gt;" id="2"/> in
<x equiv-text="${billingTabLink}" id="3"/>.</source>
</trans-unit>
<trans-unit id="se5578c14db3c7b2b">
<source>Your web archives are always yours — you can download any archived items you'd like to keep
before the trial ends!</source>
</trans-unit>
<trans-unit id="hc4152410e53b56c9">
<source>To choose a plan and continue using Browsertrix, see
<x equiv-text="${billingTabLink}" id="0"/>.</source>
</trans-unit>
<trans-unit id="h003bd6a4e60ee0a5">
<source>Your trial will end on <x equiv-text="${futureCancelDate}" id="0"/>
- Click <x equiv-text="&lt;strong&gt;" id="1"/>Choose Plan<x equiv-text="&lt;/strong&gt;" id="2"/> to
subscribe</source>
</trans-unit>
<trans-unit id="h244d3ee006a72650">
<source>Your plan will be canceled on
<x equiv-text="${futureCancelDate}" id="0"/></source>
</trans-unit>
</body>
</file>
</xliff>