00eb62214d
* basecrawl refactor: make crawls db more generic, supporting different types of 'base crawls': crawls, uploads, manual archives - move shared functionality to basecrawl.py - create a base BaseCrawl object, which contains start / finish time, metadata and files array - create BaseCrawlOps, base class for CrawlOps, which supports base crawl deletion, querying and collection add/remove * uploads api: (part of #929) - new UploadCrawl object which extends BaseCrawl, has name and description - support multipart form data data upload to /uploads/formdata - support streaming upload of a single file via /uploads/stream, using botocore multipart upload to upload to s3-endpoint in parts - require 'filename' param to set upload filename for streaming uploads (otherwise use form data names) - sanitize filename, place uploads in /uploads/<uuid>/<sanitized-filename>-<random>.wacz - uploads have internal id 'upload-<uuid>' - create UploadedCrawl object with CrawlFiles pointing to the newly uploaded files, set state to 'complete' - handle upload failures, abort multipart upload - ensure uploads added within org bucket path - return id / added when adding new UploadedCrawl - support listing, deleting, and patch /uploads - support upload details via /replay.json to support for replay - add support for 'replaceId=<id>', which would remove all previous files in upload after new upload succeeds. if replaceId doesn't exist, create new upload. (only for stream endpoint so far). - support patching upload metadata: notes, tags and name on uploads (UpdateUpload extends UpdateCrawl and adds 'name') * base crawls api: Add /all-crawls list and delete endpoints for all crawl types (without resources) - support all-crawls/<id>/replay.json with resources - Use ListCrawlOut model for /all-crawls list endpoint - Extend BaseCrawlOut from ListCrawlOut, add type - use 'type: crawl' for crawls and 'type: upload' for uploads - migration: ensure all previous crawl objects / missing type are set to 'type: crawl' - indexes: add db indices on 'type' field and with 'type' field and oid, cid, finished, state * tests: add test for multipart and streaming upload, listing uploads, deleting upload - add sample WACZ for upload testing: 'example.wacz' and 'example-2.wacz' * collections: support adding and remove both crawls and uploads via base crawl - include collection_ids in /all-crawls list - collections replay.json can include both crawls and uploads bump version to 1.6.0-beta.2 --------- Co-authored-by: Tessa Walsh <tessa@bitarchivist.net>
135 lines
3.7 KiB
JSON
135 lines
3.7 KiB
JSON
{
|
|
"name": "browsertrix-frontend",
|
|
"version": "1.6.0-beta.2",
|
|
"main": "index.ts",
|
|
"license": "AGPL-3.0-or-later",
|
|
"dependencies": {
|
|
"@cheap-glitch/mi-cron": "^1.0.1",
|
|
"@lit/localize": "^0.11.4",
|
|
"@novnc/novnc": "^1.4.0-beta",
|
|
"@rollup/plugin-commonjs": "^18.0.0",
|
|
"@shoelace-style/shoelace": "^2.4.0",
|
|
"@types/color": "^3.0.2",
|
|
"@types/lodash": "^4.14.178",
|
|
"@types/sinon": "^10.0.6",
|
|
"@typescript-eslint/eslint-plugin": "^5.4.0",
|
|
"@typescript-eslint/parser": "^5.4.0",
|
|
"@wysimark/standalone": "2.2.15",
|
|
"@xstate/fsm": "^1.6.2",
|
|
"autoprefixer": "^10.4.2",
|
|
"axios": "^0.22.0",
|
|
"broadcastchannel-polyfill": "^1.0.1",
|
|
"color": "^4.0.1",
|
|
"copy-webpack-plugin": "^9.1.0",
|
|
"css-loader": "^6.3.0",
|
|
"del-cli": "^4.0.1",
|
|
"dotenv": "^10.0.0",
|
|
"dotenv-webpack": "^7.0.3",
|
|
"eslint": "^8.2.0",
|
|
"eslint-config-prettier": "^8.3.0",
|
|
"eslint-plugin-lit": "^1.6.1",
|
|
"eslint-plugin-wc": "^1.3.2",
|
|
"eslint-webpack-plugin": "^3.1.1",
|
|
"fuse.js": "^6.5.3",
|
|
"glob": "^8.1.0",
|
|
"html-loader": "^3.0.1",
|
|
"html-webpack-plugin": "^5.5.0",
|
|
"immutable": "^4.1.0",
|
|
"iso-639-1": "^2.1.15",
|
|
"lit": "^2.4.1",
|
|
"lodash": "^4.17.21",
|
|
"micromark": "^3.1.0",
|
|
"node-fetch": "^3.1.0",
|
|
"postcss": "^8.4.5",
|
|
"postcss-loader": "^6.1.1",
|
|
"prettier": "^2.4.1",
|
|
"pretty-ms": "^7.0.1",
|
|
"query-string": "^8.1.0",
|
|
"regex-colorize": "^0.0.3",
|
|
"style-loader": "^3.3.0",
|
|
"tailwindcss": "^3.2.7",
|
|
"ts-loader": "^9.2.6",
|
|
"typescript": "^4.5.2",
|
|
"update-dotenv": "^1.1.1",
|
|
"url-pattern": "^1.0.3",
|
|
"webpack": "^5.56.0",
|
|
"webpack-cli": "^4.8.0",
|
|
"webpack-merge": "^5.8.0",
|
|
"yaml": "^2.0.0-11"
|
|
},
|
|
"scripts": {
|
|
"prepare": "cd .. && husky install frontend/.husky",
|
|
"test": "web-test-runner \"src/**/*.test.{ts,js}\" --playwright --browsers chromium",
|
|
"prebuild": "del-cli ./dist",
|
|
"build": "webpack --config webpack.prod.js",
|
|
"build-dev": "webpack --mode development",
|
|
"start": "webpack serve --mode=development --config webpack.dev.js",
|
|
"serve": "node scripts/serve.js",
|
|
"lint": "eslint --fix \"src/**/*.{ts,js}\"",
|
|
"format": "prettier --write \"src/**/*.{ts,js,html,css,json}\"",
|
|
"localize:prepare": "yarn localize:extract && yarn localize:build",
|
|
"localize:extract": "lit-localize extract",
|
|
"localize:build": "lit-localize build"
|
|
},
|
|
"devDependencies": {
|
|
"@esm-bundle/chai": "^4.3.4-fix.0",
|
|
"@lit/localize-tools": "^0.6.5",
|
|
"@open-wc/testing": "^3.1.7",
|
|
"@playwright/test": "1.32.1",
|
|
"@web/dev-server-esbuild": "^0.3.3",
|
|
"@web/dev-server-import-maps": "^0.0.6",
|
|
"@web/dev-server-rollup": "^0.3.21",
|
|
"@web/test-runner": "^0.13.22",
|
|
"@web/test-runner-playwright": "^0.8.8",
|
|
"chromium": "^3.0.3",
|
|
"husky": "^8.0.3",
|
|
"lint-staged": "^13.1.0",
|
|
"sinon": "^12.0.1",
|
|
"webpack-dev-server": "^4.3.0"
|
|
},
|
|
"eslintConfig": {
|
|
"parser": "@typescript-eslint/parser",
|
|
"env": {
|
|
"browser": true,
|
|
"commonjs": true,
|
|
"es2017": true
|
|
},
|
|
"extends": [
|
|
"plugin:wc/recommended",
|
|
"plugin:lit/recommended",
|
|
"prettier"
|
|
],
|
|
"plugins": [
|
|
"@typescript-eslint",
|
|
"lit"
|
|
],
|
|
"rules": {
|
|
"no-restricted-globals": [
|
|
2,
|
|
"event",
|
|
"error"
|
|
],
|
|
"no-unused-vars": "warn"
|
|
}
|
|
},
|
|
"prettier": {
|
|
"tabWidth": 2,
|
|
"useTabs": false
|
|
},
|
|
"lint-staged": {
|
|
"src/**/*.{ts,js}": "eslint --fix --quiet",
|
|
"src/*.{ts,js,html,css,json}": "prettier --write"
|
|
},
|
|
"husky": {
|
|
"hooks": {
|
|
"pre-commit": ".husky/pre-commit"
|
|
}
|
|
},
|
|
"engines": {
|
|
"node": ">=16 <20"
|
|
},
|
|
"resolutions": {
|
|
"**/playwright": "1.32.1"
|
|
}
|
|
}
|