Reformat backend for black 23.1.0 (#548)
This commit is contained in:
parent
58aafc4191
commit
23022193fb
@ -521,7 +521,6 @@ class CrawlConfigOps:
|
|||||||
status = "deleted"
|
status = "deleted"
|
||||||
|
|
||||||
else:
|
else:
|
||||||
|
|
||||||
if not await self.crawl_configs.find_one_and_update(
|
if not await self.crawl_configs.find_one_and_update(
|
||||||
{"_id": crawlconfig.id, "inactive": {"$ne": True}},
|
{"_id": crawlconfig.id, "inactive": {"$ne": True}},
|
||||||
{"$set": query},
|
{"$set": query},
|
||||||
@ -656,7 +655,6 @@ def init_crawl_config_api(
|
|||||||
|
|
||||||
@router.delete("/{cid}")
|
@router.delete("/{cid}")
|
||||||
async def make_inactive(cid: str, org: Organization = Depends(org_crawl_dep)):
|
async def make_inactive(cid: str, org: Organization = Depends(org_crawl_dep)):
|
||||||
|
|
||||||
crawlconfig = await ops.get_crawl_config(uuid.UUID(cid), org)
|
crawlconfig = await ops.get_crawl_config(uuid.UUID(cid), org)
|
||||||
|
|
||||||
if not crawlconfig:
|
if not crawlconfig:
|
||||||
|
@ -687,7 +687,6 @@ def init_crawls_api(app, mdb, users, crawl_manager, crawl_config_ops, orgs, user
|
|||||||
async def scale_crawl(
|
async def scale_crawl(
|
||||||
scale: CrawlScale, crawl_id, org: Organization = Depends(org_crawl_dep)
|
scale: CrawlScale, crawl_id, org: Organization = Depends(org_crawl_dep)
|
||||||
):
|
):
|
||||||
|
|
||||||
result = await crawl_manager.scale_crawl(crawl_id, org.id_str, scale.scale)
|
result = await crawl_manager.scale_crawl(crawl_id, org.id_str, scale.scale)
|
||||||
if not result or not result.get("success"):
|
if not result or not result.get("success"):
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
@ -740,7 +739,6 @@ def init_crawls_api(app, mdb, users, crawl_manager, crawl_config_ops, orgs, user
|
|||||||
org: Organization = Depends(org_crawl_dep),
|
org: Organization = Depends(org_crawl_dep),
|
||||||
user: User = Depends(user_dep),
|
user: User = Depends(user_dep),
|
||||||
):
|
):
|
||||||
|
|
||||||
return await ops.add_exclusion(crawl_id, regex, org, user)
|
return await ops.add_exclusion(crawl_id, regex, org, user)
|
||||||
|
|
||||||
@app.delete(
|
@app.delete(
|
||||||
@ -753,7 +751,6 @@ def init_crawls_api(app, mdb, users, crawl_manager, crawl_config_ops, orgs, user
|
|||||||
org: Organization = Depends(org_crawl_dep),
|
org: Organization = Depends(org_crawl_dep),
|
||||||
user: User = Depends(user_dep),
|
user: User = Depends(user_dep),
|
||||||
):
|
):
|
||||||
|
|
||||||
return await ops.remove_exclusion(crawl_id, regex, org, user)
|
return await ops.remove_exclusion(crawl_id, regex, org, user)
|
||||||
|
|
||||||
return ops
|
return ops
|
||||||
|
@ -6,6 +6,7 @@ import ssl
|
|||||||
|
|
||||||
from email.message import EmailMessage
|
from email.message import EmailMessage
|
||||||
|
|
||||||
|
|
||||||
# pylint: disable=too-few-public-methods
|
# pylint: disable=too-few-public-methods
|
||||||
class EmailSender:
|
class EmailSender:
|
||||||
"""SMTP Email Sender"""
|
"""SMTP Email Sender"""
|
||||||
|
@ -17,7 +17,7 @@ class K8SCrawlJob(K8SJobMixin, CrawlJob):
|
|||||||
async def _do_scale(self, new_scale):
|
async def _do_scale(self, new_scale):
|
||||||
crawl = await self._get_crawl()
|
crawl = await self._get_crawl()
|
||||||
if not crawl:
|
if not crawl:
|
||||||
raise Exception("crawl_not_found")
|
raise RuntimeError("crawl_not_found")
|
||||||
|
|
||||||
# if making scale smaller, ensure existing crawlers saved their data
|
# if making scale smaller, ensure existing crawlers saved their data
|
||||||
pods = []
|
pods = []
|
||||||
|
@ -4,6 +4,7 @@ from kubernetes_asyncio import client, config
|
|||||||
from kubernetes_asyncio.stream import WsApiClient
|
from kubernetes_asyncio.stream import WsApiClient
|
||||||
from kubernetes_asyncio.client.api_client import ApiClient
|
from kubernetes_asyncio.client.api_client import ApiClient
|
||||||
|
|
||||||
|
|
||||||
# pylint: disable=too-few-public-methods
|
# pylint: disable=too-few-public-methods
|
||||||
class K8sAPI:
|
class K8sAPI:
|
||||||
"""K8S API accessors"""
|
"""K8S API accessors"""
|
||||||
|
@ -198,8 +198,9 @@ class K8SManager(BaseCrawlManager, K8sAPI):
|
|||||||
f"storage-{storage_name}",
|
f"storage-{storage_name}",
|
||||||
namespace=self.namespace,
|
namespace=self.namespace,
|
||||||
)
|
)
|
||||||
|
# pylint: disable=broad-except
|
||||||
except Exception:
|
except Exception:
|
||||||
# pylint: disable=broad-except,raise-missing-from
|
# pylint: disable=broad-exception-raised,raise-missing-from
|
||||||
raise Exception(f"Storage {storage_name} not found")
|
raise Exception(f"Storage {storage_name} not found")
|
||||||
|
|
||||||
return None
|
return None
|
||||||
|
@ -315,6 +315,7 @@ class OrgOps:
|
|||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
# pylint: disable=too-many-statements
|
||||||
def init_orgs_api(app, mdb, user_manager, invites, user_dep: User):
|
def init_orgs_api(app, mdb, user_manager, invites, user_dep: User):
|
||||||
"""Init organizations api router for /orgs"""
|
"""Init organizations api router for /orgs"""
|
||||||
# pylint: disable=too-many-locals
|
# pylint: disable=too-many-locals
|
||||||
@ -422,7 +423,6 @@ def init_orgs_api(app, mdb, user_manager, invites, user_dep: User):
|
|||||||
org: Organization = Depends(org_owner_dep),
|
org: Organization = Depends(org_owner_dep),
|
||||||
user: User = Depends(user_dep),
|
user: User = Depends(user_dep),
|
||||||
):
|
):
|
||||||
|
|
||||||
other_user = await user_manager.user_db.get_by_email(update.email)
|
other_user = await user_manager.user_db.get_by_email(update.email)
|
||||||
if not other_user:
|
if not other_user:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
@ -443,7 +443,6 @@ def init_orgs_api(app, mdb, user_manager, invites, user_dep: User):
|
|||||||
org: Organization = Depends(org_owner_dep),
|
org: Organization = Depends(org_owner_dep),
|
||||||
user: User = Depends(user_dep),
|
user: User = Depends(user_dep),
|
||||||
):
|
):
|
||||||
|
|
||||||
if await invites.invite_user(
|
if await invites.invite_user(
|
||||||
invite,
|
invite,
|
||||||
user,
|
user,
|
||||||
|
@ -20,6 +20,7 @@ from .crawlconfigs import CrawlConfigIdNameOut
|
|||||||
|
|
||||||
BROWSER_EXPIRE = 300
|
BROWSER_EXPIRE = 300
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
class ProfileFile(BaseModel):
|
class ProfileFile(BaseModel):
|
||||||
"""file from a crawl"""
|
"""file from a crawl"""
|
||||||
|
@ -99,7 +99,7 @@ async def get_presigned_url(org, crawlfile, crawl_manager, duration=3600):
|
|||||||
s3storage = org.storage
|
s3storage = org.storage
|
||||||
|
|
||||||
else:
|
else:
|
||||||
raise Exception("No Default Storage Found, Invalid Storage Type")
|
raise TypeError("No Default Storage Found, Invalid Storage Type")
|
||||||
|
|
||||||
async with get_s3_client(s3storage, s3storage.use_access_for_presign) as (
|
async with get_s3_client(s3storage, s3storage.use_access_for_presign) as (
|
||||||
client,
|
client,
|
||||||
|
@ -475,7 +475,6 @@ def init_users_api(app, user_manager):
|
|||||||
async def get_existing_user_invite_info(
|
async def get_existing_user_invite_info(
|
||||||
token: str, user: User = Depends(current_active_user)
|
token: str, user: User = Depends(current_active_user)
|
||||||
):
|
):
|
||||||
|
|
||||||
try:
|
try:
|
||||||
invite = user.invites[token]
|
invite = user.invites[token]
|
||||||
except:
|
except:
|
||||||
|
Loading…
Reference in New Issue
Block a user