From f157e2031f4b1d4adbf843966ac0ae32c419f9fa Mon Sep 17 00:00:00 2001 From: sua yoo Date: Sat, 23 Apr 2022 20:11:53 -0700 Subject: [PATCH] Filter and sort crawl templates (#217) --- .../pages/archive/browser-profiles-list.ts | 2 +- .../src/pages/archive/crawl-templates-list.ts | 515 ++++++++++++------ frontend/src/pages/archive/crawls-list.ts | 7 +- 3 files changed, 347 insertions(+), 177 deletions(-) diff --git a/frontend/src/pages/archive/browser-profiles-list.ts b/frontend/src/pages/archive/browser-profiles-list.ts index acd68d3c..5f461ac6 100644 --- a/frontend/src/pages/archive/browser-profiles-list.ts +++ b/frontend/src/pages/archive/browser-profiles-list.ts @@ -49,7 +49,7 @@ export class BrowserProfilesList extends LiteElement { return html`
diff --git a/frontend/src/pages/archive/crawl-templates-list.ts b/frontend/src/pages/archive/crawl-templates-list.ts index 443326d4..a6bbdc5e 100644 --- a/frontend/src/pages/archive/crawl-templates-list.ts +++ b/frontend/src/pages/archive/crawl-templates-list.ts @@ -2,6 +2,12 @@ import type { HTMLTemplateResult } from "lit"; import { state, property } from "lit/decorators.js"; import { msg, localized, str } from "@lit/localize"; import cronParser from "cron-parser"; +import debounce from "lodash/fp/debounce"; +import flow from "lodash/fp/flow"; +import map from "lodash/fp/map"; +import orderBy from "lodash/fp/orderBy"; +import filter from "lodash/fp/filter"; +import Fuse from "fuse.js"; import type { AuthState } from "../../utils/AuthService"; import LiteElement, { html } from "../../utils/LiteElement"; @@ -14,6 +20,14 @@ type RunningCrawlsMap = { [configId: string]: string; }; +const MIN_SEARCH_LENGTH = 2; +const sortableFieldLabels = { + created_desc: msg("Newest"), + created_asc: msg("Oldest"), + lastCrawlTime_desc: msg("Newest Crawl"), + lastCrawlTime_asc: msg("Oldest Crawl"), +}; + /** * Usage: * ```ts @@ -40,9 +54,34 @@ export class CrawlTemplatesList extends LiteElement { @state() selectedTemplateForEdit?: CrawlTemplate; + @state() + private orderBy: { + field: "created"; + direction: "asc" | "desc"; + } = { + field: "created", + direction: "desc", + }; + + @state() + private searchBy: string = ""; + + @state() + private filterByScheduled: boolean | null = null; + + // For fuzzy search: + private fuse = new Fuse([], { + keys: ["name"], + shouldSort: false, + threshold: 0.4, // stricter; default is 0.6 + }); + async firstUpdated() { try { this.crawlTemplates = await this.getCrawlTemplates(); + + // Update search/filter collection + this.fuse.setCollection(this.crawlTemplates as any); } catch (e) { this.notify({ message: msg("Sorry, couldn't retrieve crawl templates at this time."), @@ -53,182 +92,24 @@ export class CrawlTemplatesList extends LiteElement { } render() { - if (!this.crawlTemplates) { - return html`
- -
`; - } - return html` -
- - ${msg("New Crawl Template")} - -
+
${this.renderControls()}
-
- ${this.crawlTemplates.map( - (t) => - html` -
-
- ${t.name} -
- - ${this.renderCardMenu(t)} -
- -
-
-
- - typeof seed === "string" ? seed : seed.url - ) - .join(", ")} - > -
- ${t.config.seeds - .map((seed) => - typeof seed === "string" ? seed : seed.url - ) - .join(", ")} -
- -
-
- ${t.crawlCount === 1 - ? msg(str`${t.crawlCount} crawl`) - : msg( - str`${(t.crawlCount || 0).toLocaleString()} crawls` - )} -
- -
- ${t.schedule - ? html` - - - - - - ` - : html`${msg("No schedule")}`} -
-
- ${this.renderCardFooter(t)} + ${this.crawlTemplates + ? this.crawlTemplates.length + ? this.renderTemplateList() + : html` +
+

+ ${msg("No crawl templates yet.")} +

- ` - )} -
+ ` + : html`
+ +
`} +
+ + + +
+ +
+ + ${msg("New Crawl Template")} + +
+
+ + ${this.crawlTemplates && this.crawlTemplates.length + ? html`
+
+ + + +
+
+
+ ${msg("Sort By")} +
+ { + const [field, direction] = e.detail.item.value.split("_"); + this.orderBy = { + field: field, + direction: direction, + }; + }} + > + ${(sortableFieldLabels as any)[this.orderBy.field] || + sortableFieldLabels[ + `${this.orderBy.field}_${this.orderBy.direction}` + ]} + + ${Object.entries(sortableFieldLabels).map( + ([value, label]) => html` + ${label} + ` + )} + + + { + this.orderBy = { + ...this.orderBy, + direction: + this.orderBy.direction === "asc" ? "desc" : "asc", + }; + }} + > +
+
` + : ""} + `; + } + + private renderTemplateList() { + const flowFns = [ + orderBy(this.orderBy.field, this.orderBy.direction), + map(this.renderTemplateItem.bind(this)), + ]; + + if (this.filterByScheduled === true) { + flowFns.unshift(filter(({ schedule }: any) => Boolean(schedule))); + } else if (this.filterByScheduled === false) { + flowFns.unshift(filter(({ schedule }: any) => !schedule)); + } + + if (this.searchBy.length >= MIN_SEARCH_LENGTH) { + flowFns.unshift(this.filterResults); + } + + return html` +
+ ${flow(...flowFns)(this.crawlTemplates)} +
+ `; + } + + private renderTemplateItem(t: CrawlTemplate) { + return html` +
+
+ ${t.name} +
+ + ${this.renderCardMenu(t)} +
+ +
+
+
+ (typeof seed === "string" ? seed : seed.url)) + .join(", ")} + > +
+ ${t.config.seeds + .map((seed) => (typeof seed === "string" ? seed : seed.url)) + .join(", ")} +
+ +
+
+ ${t.crawlCount === 1 + ? msg(str`${t.crawlCount} crawl`) + : msg(str`${(t.crawlCount || 0).toLocaleString()} crawls`)} +
+
+ ${t.crawlCount + ? html` + + ${msg( + str`Last Crawl: ${t.lastCrawlState && t.lastCrawlState.replace(/_/g, " ")}` + )} + + { + e.stopPropagation(); + this.navLink(e); + }} + > + + + ` + : html` + ${msg("No finished crawls")} + `} +
+
+ ${t.schedule + ? html` + + + + + + ` + : html`${msg("No schedule")}`} +
+
+ ${this.renderCardFooter(t)} +
+ `; + } + private renderCardMenu(t: CrawlTemplate) { const menuItems: HTMLTemplateResult[] = [ html` @@ -383,6 +542,16 @@ export class CrawlTemplatesList extends LiteElement { `; } + private onSearchInput = debounce(200)((e: any) => { + this.searchBy = e.target.value; + }) as any; + + private filterResults = () => { + const results = this.fuse.search(this.searchBy); + + return results.map(({ item }) => item); + }; + /** * Fetch crawl templates and record running crawls * associated with the crawl templates diff --git a/frontend/src/pages/archive/crawls-list.ts b/frontend/src/pages/archive/crawls-list.ts index 6ada5765..b390d454 100644 --- a/frontend/src/pages/archive/crawls-list.ts +++ b/frontend/src/pages/archive/crawls-list.ts @@ -129,7 +129,7 @@ export class CrawlsList extends LiteElement { return html`
-
${this.renderControls()}
+
${this.renderControls()}
${this.crawls.length ? this.renderCrawlList() @@ -164,10 +164,10 @@ export class CrawlsList extends LiteElement {
- ${msg("Sort by")} + ${msg("Sort By")}