) {
if (
(changedProperties.has("workflowId") && this.workflowId) ||
(changedProperties.get("isEditing") === true && this.isEditing === false)
) {
this.fetchWorkflow();
}
if (changedProperties.has("isEditing") && this.isEditing) {
this.stopPoll();
}
if (
changedProperties.get("lastCrawlId") &&
!this.lastCrawlId &&
this.activePanel === "watch"
) {
this.handleCrawlRunEnd();
}
if (
!this.isEditing &&
changedProperties.has("activePanel") &&
this.activePanel
) {
if (!this.isPanelHeaderVisible) {
// Scroll panel header into view
this.querySelector("btrix-tab-list")?.scrollIntoView({
behavior: "smooth",
});
}
if (this.activePanel === "artifacts") {
this.fetchCrawls();
}
}
}
private getActivePanelFromHash = async () => {
await this.updateComplete;
if (this.isEditing) return;
const hashValue = window.location.hash.slice(1);
if (SECTIONS.includes(hashValue as any)) {
this.activePanel = hashValue as Tab;
} else {
this.goToTab(DEFAULT_SECTION, { replace: true });
}
};
private goToTab(tab: Tab, { replace = false } = {}) {
const path = `${window.location.href.split("#")[0]}#${tab}`;
if (replace) {
window.history.replaceState(null, "", path);
} else {
window.history.pushState(null, "", path);
}
this.activePanel = tab;
}
private async handleCrawlRunEnd() {
this.goToTab("artifacts", { replace: true });
await this.fetchWorkflow();
let notifyOpts = {
message: msg("Crawl finished."),
variant: "info",
icon: "info-circle",
} as any;
// TODO consolidate with `CrawlStatus.getContent`
switch (this.workflow!.lastCrawlState) {
case "complete":
notifyOpts = {
message: msg("Crawl complete."),
variant: "success",
icon: "check-circle",
};
break;
case "canceled":
notifyOpts = {
message: msg("Crawl canceled."),
variant: "danger",
icon: "x-octagon",
};
break;
case "failed":
notifyOpts = {
message: msg("Crawl failed."),
variant: "danger",
icon: "exclamation-triangle",
};
break;
default:
break;
}
this.notify({
...notifyOpts,
duration: 8000,
});
}
private async fetchWorkflow() {
this.stopPoll();
this.isLoading = true;
try {
this.getWorkflowPromise = this.getWorkflow();
this.workflow = await this.getWorkflowPromise;
this.lastCrawlId = this.workflow.lastCrawlId;
this.lastCrawlStartTime = this.workflow.lastCrawlStartTime;
if (this.lastCrawlId) {
this.fetchCurrentCrawlStats();
}
} catch (e: any) {
this.notify({
message:
e.statusCode === 404
? msg("Workflow not found.")
: msg("Sorry, couldn't retrieve Workflow at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
this.isLoading = false;
if (!this.isEditing) {
// Restart timer for next poll
this.timerId = window.setTimeout(() => {
this.fetchWorkflow();
}, 1000 * POLL_INTERVAL_SECONDS);
}
}
render() {
if (this.isEditing && this.isCrawler) {
return html`
${when(this.workflow, this.renderEditor)}
`;
}
return html`
${this.renderHeader()}
${when(
this.workflow,
this.renderTabList,
() => html`
`
)}
(this.openDialogName = undefined)}
@sl-show=${this.showDialog}
@sl-after-hide=${() => (this.isDialogVisible = false)}
>
${msg(
"Pages crawled so far will be saved and marked as incomplete. Are you sure you want to stop crawling?"
)}
(this.openDialogName = undefined)}
>Keep Crawling
{
await this.stop();
this.openDialogName = undefined;
}}
>Stop Crawling
(this.openDialogName = undefined)}
@sl-show=${this.showDialog}
@sl-after-hide=${() => (this.isDialogVisible = false)}
>
${msg(
"Canceling will discard all pages crawled. Are you sure you want to discard them?"
)}
(this.openDialogName = undefined)}
>Keep Crawling
{
await this.cancel();
this.openDialogName = undefined;
}}
>Cancel & Discard Crawl
`;
}
private renderHeader(workflowId?: string) {
return html`
${workflowId
? msg(html`Back to ${this.renderName()}`)
: msg("Back to Crawl Workflows")}
`;
}
private renderTabList = () => html`
(this.isPanelHeaderVisible = detail.entry.isIntersecting)}
>
${this.renderPanelHeader()}
${this.renderTab("artifacts")}
${this.renderTab("watch", { disabled: !this.lastCrawlId })}
${this.renderTab("settings")}
${this.renderArtifacts()}
${until(
this.getWorkflowPromise?.then(
() => html`
${when(this.activePanel === "watch", () =>
this.workflow?.isCrawlRunning
? html`
${this.renderCurrentCrawl()}
${this.renderWatchCrawl()}`
: this.renderInactiveWatchCrawl()
)}
`
)
)}
${this.renderSettings()}
`;
private renderPanelHeader() {
if (!this.activePanel) return;
if (this.activePanel === "artifacts") {
return html`
${this.tabLabels[this.activePanel]}
${when(
this.crawls,
() =>
html`
(${this.crawls!.total.toLocaleString()}${this.workflow
?.isCrawlRunning
? html` + 1 `
: ""})
`
)}
`;
}
if (this.activePanel === "settings") {
return html` ${this.tabLabels[this.activePanel]}
this.navTo(
`/orgs/${this.workflow?.oid}/workflows/crawl/${this.workflow?.id}?edit`
)}
>
`;
}
if (this.activePanel === "watch") {
return html` ${this.tabLabels[this.activePanel]}
(this.openDialogName = "scale")}
>
${msg("Edit Instances")}
`;
}
return html`${this.tabLabels[this.activePanel]} `;
}
private renderTab(tabName: Tab, { disabled = false } = {}) {
const isActive = tabName === this.activePanel;
let className = "text-neutral-600 hover:bg-neutral-50";
if (isActive) {
className = "text-blue-600 bg-blue-50 shadow-sm";
} else if (disabled) {
className = "text-neutral-300 cursor-not-allowed";
}
return html`
{
if (disabled) e.preventDefault();
}}
>
${this.tabLabels[tabName]}
`;
}
private renderEditor = () => html`
${this.renderHeader(this.workflow!.id)}
${when(
!this.isLoading,
() => html`
this.navTo(
`/orgs/${this.orgId}/workflows/crawl/${this.workflow!.id}`
)}
>
`
)}
`;
private renderActions = () => {
if (!this.workflow) return;
const workflow = this.workflow;
return html`
${when(
this.workflow?.isCrawlRunning,
() => html`
(this.openDialogName = "stop")}
?disabled=${!this.lastCrawlId ||
this.isCancelingOrStoppingCrawl ||
this.workflow?.lastCrawlStopping}
>
${msg("Stop")}
(this.openDialogName = "cancel")}
?disabled=${!this.lastCrawlId || this.isCancelingOrStoppingCrawl}
>
${msg("Cancel")}
`,
() => html`
this.runNow()}
>
${msg("Run Crawl")}
`
)}
${msg("Actions")}
${when(
this.workflow?.isCrawlRunning,
// HACK shoelace doesn't current have a way to override non-hover
// color without resetting the --sl-color-neutral-700 variable
() => html`
(this.openDialogName = "stop")}
?disabled=${workflow.lastCrawlStopping ||
this.isCancelingOrStoppingCrawl}
>
${msg("Stop Crawl")}
(this.openDialogName = "cancel")}
>
${msg("Cancel & Discard Crawl")}
`,
() => html`
this.runNow()}
>
${msg("Run Crawl")}
`
)}
${when(
workflow.isCrawlRunning,
() => html`
(this.openDialogName = "scale")}>
${msg("Edit Crawler Instances")}
(this.openDialogName = "exclusions")}
>
${msg("Edit Exclusions")}
`
)}
this.navTo(
`/orgs/${workflow.oid}/workflows/crawl/${workflow.id}?edit`
)}
>
${msg("Edit Workflow Settings")}
CopyButton.copyToClipboard(workflow.tags.join(", "))}
?disabled=${!workflow.tags.length}
>
${msg("Copy Tags")}
this.duplicateConfig()}>
${msg("Duplicate Workflow")}
${when(!this.lastCrawlId, () => {
const shouldDeactivate = workflow.crawlCount && !workflow.inactive;
return html`
shouldDeactivate ? this.deactivate() : this.delete()}
>
${shouldDeactivate
? msg("Deactivate Workflow")
: msg("Delete Workflow")}
`;
})}
`;
};
private renderDetails() {
return html`
${this.renderDetailItem(
msg("Status"),
() => html`
`
)}
${this.renderDetailItem(
msg("Total Size"),
() => html` `
)}
${this.renderDetailItem(msg("Schedule"), () =>
this.workflow!.schedule
? html`
${humanizeSchedule(this.workflow!.schedule, {
length: "short",
})}
`
: html`${msg("No Schedule")} `
)}
${this.renderDetailItem(
msg("Created By"),
() =>
msg(
str`${
this.workflow!.createdByName
} on ${this.dateFormatter.format(
new Date(`${this.workflow!.created}Z`)
)}`
),
true
)}
`;
}
private renderDetailItem(
label: string | TemplateResult,
renderContent: () => any,
isLast = false
) {
return html`
${when(
this.workflow,
renderContent,
() => html` `
)}
${when(!isLast, () => html` `)}
`;
}
private renderName() {
if (!this.workflow) return "";
if (this.workflow.name) return this.workflow.name;
const { config } = this.workflow;
const firstSeed = config.seeds[0];
let firstSeedURL = firstSeed.url;
if (config.seeds.length === 1) {
return firstSeedURL;
}
const remainderCount = config.seeds.length - 1;
if (remainderCount === 1) {
return msg(
html`${firstSeedURL}
+${remainderCount} URL `
);
}
return msg(
html`${firstSeedURL}
+${remainderCount} URLs `
);
}
private renderArtifacts() {
return html`
${msg("View:")}
{
const value = (e.target as SlSelect).value as CrawlState[];
await this.updateComplete;
this.filterBy = {
...this.filterBy,
state: value,
};
this.fetchCrawls();
}}
>
${inactiveCrawlStates.map(this.renderStatusMenuItem)}
${when(
this.workflow?.isCrawlRunning,
() => html``
)}
${msg("Start Time")}
${when(
this.crawls,
() =>
this.crawls!.items.map(
(crawl: Crawl) => html`
this.deleteCrawl(crawl)}
>
${msg("Delete Crawl")}
`
),
() => html`
`
)}
${when(
this.crawls && !this.crawls.items.length,
() => html`
${this.crawls?.total
? msg("No matching crawls found.")
: msg("No crawls yet.")}
`
)}
`;
}
private renderStatusMenuItem = (state: CrawlState) => {
const { icon, label } = CrawlStatus.getContent(state);
return html`${icon}${label} `;
};
private renderCurrentCrawl = () => {
const skeleton = html` `;
return html`
${this.renderDetailItem(msg("Pages Crawled"), () =>
this.lastCrawlStats
? msg(
str`${this.numberFormatter.format(
+(this.lastCrawlStats.done || 0)
)} / ${this.numberFormatter.format(
+(this.lastCrawlStats.found || 0)
)}`
)
: html` `
)}
${this.renderDetailItem(msg("Run Duration"), () =>
this.lastCrawlStartTime
? RelativeDuration.humanize(
new Date().valueOf() -
new Date(`${this.lastCrawlStartTime}Z`).valueOf()
)
: skeleton
)}
${this.renderDetailItem(msg("Crawl Size"), () =>
this.workflow
? html` `
: skeleton
)}
${this.renderDetailItem(
msg("Crawler Instances"),
() => (this.workflow ? this.workflow.scale : skeleton),
true
)}
`;
};
private renderWatchCrawl = () => {
if (!this.authState || !this.workflow?.lastCrawlState) return "";
let waitingMsg = null;
switch (this.workflow.lastCrawlState) {
case "starting":
waitingMsg = msg("Crawl starting...");
break;
case "waiting_capacity":
waitingMsg = msg(
"Crawl waiting for available resources before it can start..."
);
break;
case "waiting_org_limit":
waitingMsg = msg(
"Crawl waiting for others to finish, concurrent limit per Organization reached..."
);
break;
}
const isRunning = this.workflow.lastCrawlState === "running";
const isStopping = this.workflow.lastCrawlStopping;
const authToken = this.authState.headers.Authorization.split(" ")[1];
return html`
${waitingMsg
? html``
: isActive(this.workflow.lastCrawlState)
? html`
${isStopping
? html`
${msg("Crawl stopping...")}
`
: ""}
`
: this.renderInactiveCrawlMessage()}
${when(
isRunning,
() => html`
${this.renderExclusions()}
(this.openDialogName = undefined)}
@sl-show=${this.showDialog}
@sl-after-hide=${() => (this.isDialogVisible = false)}
>
${this.isDialogVisible ? this.renderEditScale() : ""}
`
)}
`;
};
private renderInactiveWatchCrawl() {
return html`
${msg("Crawl is not currently running.")}
${when(
this.workflow?.lastCrawlId,
() => html`
${msg("Replay Latest Crawl")}
`
)}
this.runNow()}>
${msg("Run Crawl")}
`;
}
private renderInactiveCrawlMessage() {
return html`
${msg("Crawl is not running.")}
`;
}
private renderExclusions() {
return html`
${msg("Crawl URLs")}
(this.openDialogName = "exclusions")}
>
${msg("Edit Exclusions")}
${when(
this.lastCrawlId,
() => html`
`
)}
(this.openDialogName = undefined)}
@sl-show=${this.showDialog}
@sl-after-hide=${() => (this.isDialogVisible = false)}
>
${this.workflow && this.isDialogVisible
? html` `
: ""}
(this.openDialogName = undefined)}
>${msg("Done Editing")}
`;
}
private renderEditScale() {
if (!this.workflow) return;
const scaleOptions = [
{
value: 1,
label: "1",
},
{
value: 2,
label: "2",
},
{
value: 3,
label: "3",
},
];
return html`
${scaleOptions.map(
({ value, label }) => html`
{
await this.scale(value);
this.openDialogName = undefined;
}}
?disabled=${this.isSubmittingUpdate}
>${label}
`
)}
(this.openDialogName = undefined)}
>${msg("Cancel")}
`;
}
private renderSettings() {
return html``;
}
private showDialog = async () => {
await this.getWorkflowPromise;
this.isDialogVisible = true;
};
private handleExclusionChange(e: CustomEvent) {
this.fetchWorkflow();
}
private async scale(value: Crawl["scale"]) {
if (!this.lastCrawlId) return;
this.isSubmittingUpdate = true;
try {
const data = await this.apiFetch(
`/orgs/${this.orgId}/crawls/${this.lastCrawlId}/scale`,
this.authState!,
{
method: "POST",
body: JSON.stringify({ scale: +value }),
}
);
if (data.scaled) {
this.fetchWorkflow();
this.notify({
message: msg("Updated crawl scale."),
variant: "success",
icon: "check2-circle",
});
} else {
throw new Error("unhandled API response");
}
} catch {
this.notify({
message: msg("Sorry, couldn't change crawl scale at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
this.isSubmittingUpdate = false;
}
private async getWorkflow(): Promise {
const data: Workflow = await this.apiFetch(
`/orgs/${this.orgId}/crawlconfigs/${this.workflowId}`,
this.authState!
);
return data;
}
private async fetchCrawls() {
try {
this.crawls = await this.getCrawls();
} catch {
this.notify({
message: msg("Sorry, couldn't get crawls at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async getCrawls(): Promise {
const query = queryString.stringify(
{
state: this.filterBy.state || inactiveCrawlStates,
cid: this.workflowId,
sortBy: "started",
},
{
arrayFormat: "comma",
}
);
const data: APIPaginatedList = await this.apiFetch(
`/orgs/${this.orgId}/crawls?${query}`,
this.authState!
);
return data;
}
private async fetchCurrentCrawlStats() {
if (!this.lastCrawlId) return;
try {
// TODO see if API can pass stats in GET workflow
const { stats } = await this.getCrawl(this.lastCrawlId);
this.lastCrawlStats = stats;
} catch (e) {
// TODO handle error
console.debug(e);
}
}
private stopPoll() {
window.clearTimeout(this.timerId);
}
private async getCrawl(crawlId: Crawl["id"]): Promise {
const data = await this.apiFetch(
`/orgs/${this.orgId}/crawls/${crawlId}/replay.json`,
this.authState!
);
return data;
}
/**
* Create a new template using existing template data
*/
private async duplicateConfig() {
if (!this.workflow) return;
const workflowParams: WorkflowParams = {
...this.workflow,
name: msg(str`${this.renderName()} Copy`),
};
this.navTo(
`/orgs/${this.orgId}/workflows?new&jobType=${workflowParams.jobType}`,
{
workflow: workflowParams,
}
);
this.notify({
message: msg(str`Copied Workflow to new template.`),
variant: "success",
icon: "check2-circle",
});
}
private async deactivate(): Promise {
if (!this.workflow) return;
try {
await this.apiFetch(
`/orgs/${this.orgId}/crawlconfigs/${this.workflow.id}`,
this.authState!,
{
method: "DELETE",
}
);
this.workflow = {
...this.workflow,
inactive: true,
};
this.notify({
message: msg(html`Deactivated ${this.renderName()} .`),
variant: "success",
icon: "check2-circle",
});
} catch {
this.notify({
message: msg("Sorry, couldn't deactivate Workflow at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async delete(): Promise {
if (!this.workflow) return;
const isDeactivating = this.workflow.crawlCount > 0;
try {
await this.apiFetch(
`/orgs/${this.orgId}/crawlconfigs/${this.workflow.id}`,
this.authState!,
{
method: "DELETE",
}
);
this.navTo(`/orgs/${this.orgId}/workflows/crawls`);
this.notify({
message: isDeactivating
? msg(html`Deactivated ${this.renderName()} .`)
: msg(html`Deleted ${this.renderName()} .`),
variant: "success",
icon: "check2-circle",
});
} catch {
this.notify({
message: isDeactivating
? msg("Sorry, couldn't deactivate Workflow at this time.")
: msg("Sorry, couldn't delete Workflow at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async cancel() {
if (!this.lastCrawlId) return;
this.isCancelingOrStoppingCrawl = true;
try {
const data = await this.apiFetch(
`/orgs/${this.orgId}/crawls/${this.lastCrawlId}/cancel`,
this.authState!,
{
method: "POST",
}
);
if (data.success === true) {
this.fetchWorkflow();
} else {
throw data;
}
} catch {
this.notify({
message: msg("Something went wrong, couldn't cancel crawl."),
variant: "danger",
icon: "exclamation-octagon",
});
}
this.isCancelingOrStoppingCrawl = false;
}
private async stop() {
if (!this.lastCrawlId) return;
this.isCancelingOrStoppingCrawl = true;
try {
const data = await this.apiFetch(
`/orgs/${this.orgId}/crawls/${this.lastCrawlId}/stop`,
this.authState!,
{
method: "POST",
}
);
if (data.success === true) {
this.fetchWorkflow();
} else {
throw data;
}
} catch {
this.notify({
message: msg("Something went wrong, couldn't stop crawl."),
variant: "danger",
icon: "exclamation-octagon",
});
}
this.isCancelingOrStoppingCrawl = false;
}
private async runNow(): Promise {
try {
const data = await this.apiFetch(
`/orgs/${this.orgId}/crawlconfigs/${this.workflow!.id}/run`,
this.authState!,
{
method: "POST",
}
);
this.lastCrawlId = data.started;
// remove 'Z' from timestamp to match API response
this.lastCrawlStartTime = new Date().toISOString().slice(0, -1);
this.fetchWorkflow();
this.goToTab("watch");
this.notify({
message: msg("Starting crawl."),
variant: "success",
icon: "check2-circle",
duration: 8000,
});
} catch {
this.notify({
message: msg("Sorry, couldn't run crawl at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
private async deleteCrawl(crawl: Crawl) {
try {
const data = await this.apiFetch(
`/orgs/${crawl.oid}/crawls/delete`,
this.authState!,
{
method: "POST",
body: JSON.stringify({
crawl_ids: [crawl.id],
}),
}
);
this.crawls = {
...this.crawls!,
items: this.crawls!.items.filter((c) => c.id !== crawl.id),
};
this.notify({
message: msg(`Successfully deleted crawl`),
variant: "success",
icon: "check2-circle",
});
this.fetchCrawls();
} catch (e: any) {
this.notify({
message:
(e.isApiError && e.message) ||
msg("Sorry, couldn't run crawl at this time."),
variant: "danger",
icon: "exclamation-octagon",
});
}
}
}
customElements.define("btrix-workflow-detail", WorkflowDetail);