From 4edc05d503e8796784397201d5f032c81db513af Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Thu, 6 Jun 2024 14:28:19 -0400 Subject: [PATCH] Use standard firstSeed/seedCount fallback for workflows with no name in profile details (#1852) Fixes #1833 - Add firstSeed and seedCount to workflow information in profile detail API endpoint (tests updated accordingly), update name of model used for limited workflow information to be more accurate - Fix name display in Crawl Workflows list at bottom of Profile detail page to be consistent with rest of application --------- Co-authored-by: Emma Segal-Grossman --- backend/btrixcloud/crawlconfigs.py | 19 ++++++---- backend/btrixcloud/models.py | 8 +++-- backend/btrixcloud/profiles.py | 14 +++----- backend/test/test_profiles.py | 2 ++ .../src/pages/org/browser-profiles-detail.ts | 36 +++++++++++++++---- frontend/src/types/crawler.ts | 9 ++++- 6 files changed, 63 insertions(+), 25 deletions(-) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index 3fb1d097..f0b165d4 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -23,7 +23,7 @@ from .models import ( ConfigRevision, CrawlConfig, CrawlConfigOut, - CrawlConfigIdNameOut, + CrawlConfigProfileOut, CrawlOut, EmptyStr, UpdateCrawlConfig, @@ -547,17 +547,24 @@ class CrawlConfigOps: return configs, total - async def get_crawl_config_ids_for_profile( - self, profileid: UUID, org: Optional[Organization] = None + async def get_crawl_config_info_for_profile( + self, profileid: UUID, org: Organization ): """Return all crawl configs that are associated with a given profileid""" query = {"profileid": profileid, "inactive": {"$ne": True}} if org: query["oid"] = org.id - cursor = self.crawl_configs.find(query, projection=["_id", "name"]) - results = await cursor.to_list(length=1000) - results = [CrawlConfigIdNameOut.from_dict(res) for res in results] + results = [] + + cursor = self.crawl_configs.find(query, projection=["_id"]) + workflows = await cursor.to_list(length=1000) + for workflow_dict in workflows: + workflow_out = await self.get_crawl_config_out( + workflow_dict.get("_id"), org + ) + results.append(CrawlConfigProfileOut.from_dict(workflow_out.to_dict())) + return results async def get_running_crawl( diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index da97376e..5553c47c 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -412,10 +412,12 @@ class CrawlConfigOut(CrawlConfigCore, CrawlConfigAdditional): # ============================================================================ -class CrawlConfigIdNameOut(BaseMongoModel): - """Crawl Config id and name output only""" +class CrawlConfigProfileOut(BaseMongoModel): + """Crawl Config basic info for profiles""" name: str + firstSeed: str + seedCount: int # ============================================================================ @@ -1197,7 +1199,7 @@ class Profile(BaseMongoModel): class ProfileWithCrawlConfigs(Profile): """Profile with list of crawlconfigs using this profile""" - crawlconfigs: List[CrawlConfigIdNameOut] = [] + crawlconfigs: List[CrawlConfigProfileOut] = [] # ============================================================================ diff --git a/backend/btrixcloud/profiles.py b/backend/btrixcloud/profiles.py index 77dd1810..68ea6a38 100644 --- a/backend/btrixcloud/profiles.py +++ b/backend/btrixcloud/profiles.py @@ -336,9 +336,7 @@ class ProfileOps: return Profile.from_dict(res) - async def get_profile_with_configs( - self, profileid: UUID, org: Optional[Organization] = None - ): + async def get_profile_with_configs(self, profileid: UUID, org: Organization): """get profile for api output, with crawlconfigs""" profile = await self.get_profile(profileid, org) @@ -369,16 +367,14 @@ class ProfileOps: except: return None - async def get_crawl_configs_for_profile( - self, profileid: UUID, org: Optional[Organization] = None - ): - """Get list of crawl config id, names for that use a particular profile""" + async def get_crawl_configs_for_profile(self, profileid: UUID, org: Organization): + """Get list of crawl configs with basic info for that use a particular profile""" - crawlconfig_names = await self.crawlconfigs.get_crawl_config_ids_for_profile( + crawlconfig_info = await self.crawlconfigs.get_crawl_config_info_for_profile( profileid, org ) - return crawlconfig_names + return crawlconfig_info async def delete_profile(self, profileid: UUID, org: Organization): """delete profile, if not used in active crawlconfig""" diff --git a/backend/test/test_profiles.py b/backend/test/test_profiles.py index 8dc7e549..e7de14f5 100644 --- a/backend/test/test_profiles.py +++ b/backend/test/test_profiles.py @@ -238,6 +238,8 @@ def test_get_profile(admin_auth_headers, default_org_id, profile_id, profile_con assert len(crawl_configs) == 1 assert crawl_configs[0]["id"] == profile_config_id assert crawl_configs[0]["name"] == "Profile Test Crawl" + assert crawl_configs[0]["firstSeed"] == "https://webrecorder.net/" + assert crawl_configs[0]["seedCount"] == 1 break except: if time.monotonic() - start_time > time_limit: diff --git a/frontend/src/pages/org/browser-profiles-detail.ts b/frontend/src/pages/org/browser-profiles-detail.ts index e5f203e5..247d8da3 100644 --- a/frontend/src/pages/org/browser-profiles-detail.ts +++ b/frontend/src/pages/org/browser-profiles-detail.ts @@ -1,12 +1,12 @@ import { localized, msg, str } from "@lit/localize"; -import { html, nothing } from "lit"; +import { html, nothing, type TemplateResult } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { ifDefined } from "lit/directives/if-defined.js"; import { when } from "lit/directives/when.js"; import { capitalize } from "lodash/fp"; import queryString from "query-string"; -import type { Profile } from "./types"; +import type { Profile, ProfileWorkflow } from "./types"; import { TailwindElement } from "@/classes/TailwindElement"; import type { Dialog } from "@/components/ui/dialog"; @@ -339,17 +339,16 @@ export class BrowserProfilesDetail extends TailwindElement { if (this.profile?.crawlconfigs?.length) { return html`