Timeout (#70)
* fix: long response time due to many <a>... ... without hrefs. It's a temporary measure until it's clear how to deal with such performance issues. * perf: remove jsdom install linkedom * feat: timeout But still this timeout works only for the time of transfer of the page itself, not its processing by the server * fix: links * format
This commit is contained in:
Artemy Egorov
GitHub
parent
b805f19f78
commit
8f707c800e
@ -1,20 +1,14 @@
|
||||
{
|
||||
"env": {
|
||||
"browser": true,
|
||||
"es2021": true
|
||||
},
|
||||
"extends": [
|
||||
"eslint:recommended",
|
||||
"plugin:@typescript-eslint/recommended"
|
||||
],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": "latest",
|
||||
"sourceType": "module"
|
||||
},
|
||||
"plugins": [
|
||||
"@typescript-eslint"
|
||||
],
|
||||
"rules": {
|
||||
}
|
||||
"env": {
|
||||
"browser": true,
|
||||
"es2021": true
|
||||
},
|
||||
"extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": "latest",
|
||||
"sourceType": "module"
|
||||
},
|
||||
"plugins": ["@typescript-eslint"],
|
||||
"rules": {}
|
||||
}
|
||||
|
||||
8
.github/dependabot.yml
vendored
8
.github/dependabot.yml
vendored
@ -5,8 +5,8 @@
|
||||
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "npm" # See documentation for possible values
|
||||
directory: "/" # Location of package manifests
|
||||
target-branch: "dependabot"
|
||||
- package-ecosystem: 'npm' # See documentation for possible values
|
||||
directory: '/' # Location of package manifests
|
||||
target-branch: 'dependabot'
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
interval: 'weekly'
|
||||
|
||||
5
.prettierrc
Normal file
5
.prettierrc
Normal file
@ -0,0 +1,5 @@
|
||||
{
|
||||
"trailingComma": "es5",
|
||||
"tabWidth": 2,
|
||||
"singleQuote": true
|
||||
}
|
||||
@ -1,10 +1,10 @@
|
||||
version: "3"
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
txtdot:
|
||||
image: ghcr.io/txtdot/txtdot:latest
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- '8080:8080'
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- ".env:/app/.env"
|
||||
- '.env:/app/.env'
|
||||
|
||||
421
package-lock.json
generated
421
package-lock.json
generated
@ -21,8 +21,8 @@
|
||||
"fastify": "^4.21.0",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"ip-range-check": "^0.2.0",
|
||||
"jsdom": "^22.1.0",
|
||||
"json-schema-to-ts": "^2.9.2",
|
||||
"linkedom": "^0.16.4",
|
||||
"micromatch": "^4.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
@ -314,14 +314,6 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@tootallnate/once": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
|
||||
"integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/braces": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.4.tgz",
|
||||
@ -590,11 +582,6 @@
|
||||
"integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/abab": {
|
||||
"version": "2.0.6",
|
||||
"resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
|
||||
"integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA=="
|
||||
},
|
||||
"node_modules/abort-controller": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
||||
@ -632,17 +619,6 @@
|
||||
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/agent-base": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
|
||||
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
|
||||
"dependencies": {
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "6.12.6",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
|
||||
@ -822,6 +798,11 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
@ -1096,30 +1077,37 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/cssstyle": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz",
|
||||
"integrity": "sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==",
|
||||
"node_modules/css-select": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
|
||||
"integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
|
||||
"dependencies": {
|
||||
"rrweb-cssom": "^0.6.0"
|
||||
"boolbase": "^1.0.0",
|
||||
"css-what": "^6.1.0",
|
||||
"domhandler": "^5.0.2",
|
||||
"domutils": "^3.0.1",
|
||||
"nth-check": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/data-urls": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz",
|
||||
"integrity": "sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==",
|
||||
"dependencies": {
|
||||
"abab": "^2.0.6",
|
||||
"whatwg-mimetype": "^3.0.0",
|
||||
"whatwg-url": "^12.0.0"
|
||||
},
|
||||
"node_modules/css-what": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
|
||||
"integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
"node": ">= 6"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
},
|
||||
"node_modules/cssom": {
|
||||
"version": "0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
|
||||
"integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
|
||||
@ -1136,11 +1124,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/decimal.js": {
|
||||
"version": "10.4.3",
|
||||
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
|
||||
"integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA=="
|
||||
},
|
||||
"node_modules/deep-is": {
|
||||
"version": "0.1.4",
|
||||
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
|
||||
@ -1187,15 +1170,42 @@
|
||||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/domexception": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz",
|
||||
"integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==",
|
||||
"node_modules/dom-serializer": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
|
||||
"integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
|
||||
"dependencies": {
|
||||
"webidl-conversions": "^7.0.0"
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.2",
|
||||
"entities": "^4.2.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/domelementtype": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
|
||||
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
]
|
||||
},
|
||||
"node_modules/domhandler": {
|
||||
"version": "5.0.3",
|
||||
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
|
||||
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
|
||||
"dependencies": {
|
||||
"domelementtype": "^2.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
"node": ">= 4"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/domhandler?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dompurify": {
|
||||
@ -1203,6 +1213,19 @@
|
||||
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.6.tgz",
|
||||
"integrity": "sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w=="
|
||||
},
|
||||
"node_modules/domutils": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
|
||||
"integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
|
||||
"dependencies": {
|
||||
"dom-serializer": "^2.0.0",
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/domutils?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "16.3.1",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
|
||||
@ -1872,15 +1895,27 @@
|
||||
"resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz",
|
||||
"integrity": "sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A=="
|
||||
},
|
||||
"node_modules/html-encoding-sniffer": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz",
|
||||
"integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==",
|
||||
"node_modules/html-escaper": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
|
||||
"integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="
|
||||
},
|
||||
"node_modules/htmlparser2": {
|
||||
"version": "9.0.0",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.0.0.tgz",
|
||||
"integrity": "sha512-uxbSI98wmFT/G4P2zXx4OVx04qWUmyFPrD2/CNepa2Zo3GPNaCaaxElDgwUrwYWkK1nr9fft0Ya8dws8coDLLQ==",
|
||||
"funding": [
|
||||
"https://github.com/fb55/htmlparser2?sponsor=1",
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/fb55"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"whatwg-encoding": "^2.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
"domelementtype": "^2.3.0",
|
||||
"domhandler": "^5.0.3",
|
||||
"domutils": "^3.1.0",
|
||||
"entities": "^4.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/http-errors": {
|
||||
@ -1898,31 +1933,6 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/http-proxy-agent": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
|
||||
"integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
|
||||
"dependencies": {
|
||||
"@tootallnate/once": "2",
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/https-proxy-agent": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
|
||||
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
|
||||
"dependencies": {
|
||||
"agent-base": "6",
|
||||
"debug": "4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
|
||||
@ -2076,11 +2086,6 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/is-potential-custom-element-name": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
|
||||
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="
|
||||
},
|
||||
"node_modules/isarray": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
|
||||
@ -2122,47 +2127,6 @@
|
||||
"js-yaml": "bin/js-yaml.js"
|
||||
}
|
||||
},
|
||||
"node_modules/jsdom": {
|
||||
"version": "22.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-22.1.0.tgz",
|
||||
"integrity": "sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==",
|
||||
"dependencies": {
|
||||
"abab": "^2.0.6",
|
||||
"cssstyle": "^3.0.0",
|
||||
"data-urls": "^4.0.0",
|
||||
"decimal.js": "^10.4.3",
|
||||
"domexception": "^4.0.0",
|
||||
"form-data": "^4.0.0",
|
||||
"html-encoding-sniffer": "^3.0.0",
|
||||
"http-proxy-agent": "^5.0.0",
|
||||
"https-proxy-agent": "^5.0.1",
|
||||
"is-potential-custom-element-name": "^1.0.1",
|
||||
"nwsapi": "^2.2.4",
|
||||
"parse5": "^7.1.2",
|
||||
"rrweb-cssom": "^0.6.0",
|
||||
"saxes": "^6.0.0",
|
||||
"symbol-tree": "^3.2.4",
|
||||
"tough-cookie": "^4.1.2",
|
||||
"w3c-xmlserializer": "^4.0.0",
|
||||
"webidl-conversions": "^7.0.0",
|
||||
"whatwg-encoding": "^2.0.0",
|
||||
"whatwg-mimetype": "^3.0.0",
|
||||
"whatwg-url": "^12.0.1",
|
||||
"ws": "^8.13.0",
|
||||
"xml-name-validator": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"canvas": "^2.5.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"canvas": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/json-buffer": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
|
||||
@ -2250,6 +2214,18 @@
|
||||
"set-cookie-parser": "^2.4.1"
|
||||
}
|
||||
},
|
||||
"node_modules/linkedom": {
|
||||
"version": "0.16.4",
|
||||
"resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.16.4.tgz",
|
||||
"integrity": "sha512-SykvDVh/jAnaO+WiPqH5vX3QpZrIRImuppzYhIHons3RXPhDwqN2dOyfopOVaHleqWtoS+3vWCqen+m8M3HToQ==",
|
||||
"dependencies": {
|
||||
"css-select": "^5.1.0",
|
||||
"cssom": "^0.5.0",
|
||||
"html-escaper": "^3.0.3",
|
||||
"htmlparser2": "^9.0.0",
|
||||
"uhyphen": "^0.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/locate-path": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
|
||||
@ -2398,10 +2374,16 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/nwsapi": {
|
||||
"version": "2.2.7",
|
||||
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz",
|
||||
"integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ=="
|
||||
"node_modules/nth-check": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
|
||||
"integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
|
||||
"dependencies": {
|
||||
"boolbase": "^1.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/fb55/nth-check?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/on-exit-leak-free": {
|
||||
"version": "2.1.2",
|
||||
@ -2486,6 +2468,7 @@
|
||||
"version": "7.1.2",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
|
||||
"integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"entities": "^4.4.0"
|
||||
},
|
||||
@ -2667,11 +2650,6 @@
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/psl": {
|
||||
"version": "1.9.0",
|
||||
"resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
|
||||
"integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
|
||||
},
|
||||
"node_modules/punycode": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
|
||||
@ -2680,11 +2658,6 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/querystringify": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
|
||||
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
|
||||
},
|
||||
"node_modules/queue-microtask": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
|
||||
@ -2764,11 +2737,6 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/requires-port": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
|
||||
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
|
||||
},
|
||||
"node_modules/resolve-from": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
|
||||
@ -2835,11 +2803,6 @@
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/rrweb-cssom": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz",
|
||||
"integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw=="
|
||||
},
|
||||
"node_modules/run-parallel": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
|
||||
@ -2903,17 +2866,6 @@
|
||||
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
|
||||
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
|
||||
},
|
||||
"node_modules/saxes": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
|
||||
"integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",
|
||||
"dependencies": {
|
||||
"xmlchars": "^2.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=v12.22.7"
|
||||
}
|
||||
},
|
||||
"node_modules/secure-json-parse": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
|
||||
@ -3091,11 +3043,6 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/symbol-tree": {
|
||||
"version": "3.2.4",
|
||||
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
|
||||
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
|
||||
},
|
||||
"node_modules/text-table": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
|
||||
@ -3189,31 +3136,6 @@
|
||||
"node": ">=0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/tough-cookie": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
|
||||
"integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
|
||||
"dependencies": {
|
||||
"psl": "^1.1.33",
|
||||
"punycode": "^2.1.1",
|
||||
"universalify": "^0.2.0",
|
||||
"url-parse": "^1.5.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz",
|
||||
"integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==",
|
||||
"dependencies": {
|
||||
"punycode": "^2.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/ts-algebra": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-1.2.2.tgz",
|
||||
@ -3289,20 +3211,17 @@
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/uhyphen": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz",
|
||||
"integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/universalify": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
|
||||
"integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
|
||||
"engines": {
|
||||
"node": ">= 4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/untildify": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz",
|
||||
@ -3320,71 +3239,12 @@
|
||||
"punycode": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/url-parse": {
|
||||
"version": "1.5.10",
|
||||
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
|
||||
"integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
|
||||
"dependencies": {
|
||||
"querystringify": "^2.1.1",
|
||||
"requires-port": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/w3c-xmlserializer": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz",
|
||||
"integrity": "sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==",
|
||||
"dependencies": {
|
||||
"xml-name-validator": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
|
||||
"integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-encoding": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz",
|
||||
"integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==",
|
||||
"dependencies": {
|
||||
"iconv-lite": "0.6.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-mimetype": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz",
|
||||
"integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "12.0.1",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz",
|
||||
"integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==",
|
||||
"dependencies": {
|
||||
"tr46": "^4.1.1",
|
||||
"webidl-conversions": "^7.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
@ -3422,39 +3282,6 @@
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.14.2",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
|
||||
"integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/xml-name-validator": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",
|
||||
"integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/xmlchars": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
|
||||
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
|
||||
},
|
||||
"node_modules/xtend": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
|
||||
|
||||
@ -17,8 +17,8 @@
|
||||
"fastify": "^4.21.0",
|
||||
"iconv-lite": "^0.6.3",
|
||||
"ip-range-check": "^0.2.0",
|
||||
"jsdom": "^22.1.0",
|
||||
"json-schema-to-ts": "^2.9.2",
|
||||
"linkedom": "^0.16.4",
|
||||
"micromatch": "^4.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
@ -41,7 +41,9 @@
|
||||
"start": "cd ./dist && node ./src/app.js",
|
||||
"start:docker": "node ./src/app.js",
|
||||
"clean-css": "cleancss --batch static/*.css -o dist/static --batch-suffix \"\"",
|
||||
"dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\""
|
||||
"dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"",
|
||||
"format": "prettier --write .",
|
||||
"format:check": "prettier --check ."
|
||||
},
|
||||
"keywords": [],
|
||||
"authors": [
|
||||
|
||||
53
src/app.ts
53
src/app.ts
@ -1,21 +1,21 @@
|
||||
import path from "path";
|
||||
import path from 'path';
|
||||
|
||||
import Fastify from "fastify";
|
||||
import fastifyStatic from "@fastify/static";
|
||||
import fastifyView from "@fastify/view";
|
||||
import fastifySwagger from "@fastify/swagger";
|
||||
import fastifySwaggerUi from "@fastify/swagger-ui";
|
||||
import ejs from "ejs";
|
||||
import Fastify from 'fastify';
|
||||
import fastifyStatic from '@fastify/static';
|
||||
import fastifyView from '@fastify/view';
|
||||
import fastifySwagger from '@fastify/swagger';
|
||||
import fastifySwaggerUi from '@fastify/swagger-ui';
|
||||
import ejs from 'ejs';
|
||||
|
||||
import indexRoute from "./routes/browser/index";
|
||||
import getRoute from "./routes/browser/get";
|
||||
import proxyRoute from "./routes/browser/proxy";
|
||||
import parseRoute from "./routes/api/parse";
|
||||
import rawHtml from "./routes/api/raw-html";
|
||||
import indexRoute from './routes/browser/index';
|
||||
import getRoute from './routes/browser/get';
|
||||
import proxyRoute from './routes/browser/proxy';
|
||||
import parseRoute from './routes/api/parse';
|
||||
import rawHtml from './routes/api/raw-html';
|
||||
|
||||
import publicConfig from "./publicConfig";
|
||||
import errorHandler from "./errors/handler";
|
||||
import getConfig from "./config/main";
|
||||
import publicConfig from './publicConfig';
|
||||
import errorHandler from './errors/handler';
|
||||
import getConfig from './config/main';
|
||||
|
||||
class App {
|
||||
async init() {
|
||||
@ -24,11 +24,12 @@ class App {
|
||||
const fastify = Fastify({
|
||||
logger: true,
|
||||
trustProxy: config.reverse_proxy,
|
||||
connectionTimeout: config.timeout,
|
||||
});
|
||||
|
||||
fastify.register(fastifyStatic, {
|
||||
root: path.join(process.cwd(), "static"),
|
||||
prefix: "/static/",
|
||||
root: path.join(process.cwd(), 'static'),
|
||||
prefix: '/static/',
|
||||
});
|
||||
|
||||
fastify.register(fastifyView, {
|
||||
@ -41,32 +42,28 @@ class App {
|
||||
await fastify.register(fastifySwagger, {
|
||||
swagger: {
|
||||
info: {
|
||||
title: "TXTDot API",
|
||||
title: 'TXTDot API',
|
||||
description: publicConfig.description,
|
||||
version: publicConfig.version,
|
||||
},
|
||||
}
|
||||
},
|
||||
});
|
||||
await fastify.register(fastifySwaggerUi, { routePrefix: "/doc" });
|
||||
await fastify.register(fastifySwaggerUi, { routePrefix: '/doc' });
|
||||
}
|
||||
|
||||
fastify.register(indexRoute);
|
||||
fastify.register(getRoute);
|
||||
|
||||
if (config.proxy_res)
|
||||
fastify.register(proxyRoute);
|
||||
if (config.proxy_res) fastify.register(proxyRoute);
|
||||
|
||||
fastify.register(parseRoute);
|
||||
fastify.register(rawHtml);
|
||||
|
||||
fastify.setErrorHandler(errorHandler);
|
||||
|
||||
fastify.listen(
|
||||
{ host: config.host, port: config.port },
|
||||
(err) => {
|
||||
err && console.log(err);
|
||||
}
|
||||
);
|
||||
fastify.listen({ host: config.host, port: config.port }, (err) => {
|
||||
err && console.log(err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
import { config } from "dotenv";
|
||||
import { config } from 'dotenv';
|
||||
|
||||
export class ConfigService {
|
||||
public readonly host: string;
|
||||
public readonly port: number;
|
||||
public readonly timeout: number;
|
||||
public readonly reverse_proxy: boolean;
|
||||
public readonly proxy_res: boolean;
|
||||
public readonly swagger: boolean;
|
||||
@ -10,9 +11,11 @@ export class ConfigService {
|
||||
constructor() {
|
||||
config();
|
||||
|
||||
this.host = process.env.HOST || "0.0.0.0";
|
||||
this.host = process.env.HOST || '0.0.0.0';
|
||||
this.port = Number(process.env.PORT) || 8080;
|
||||
|
||||
this.timeout = 1000;
|
||||
|
||||
this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false);
|
||||
|
||||
this.proxy_res = this.parseBool(process.env.PROXY_RES, true);
|
||||
@ -21,6 +24,6 @@ export class ConfigService {
|
||||
|
||||
parseBool(value: string | undefined, def: boolean): boolean {
|
||||
if (!value) return def;
|
||||
return value === "true" || value === "1";
|
||||
return value === 'true' || value === '1';
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { ConfigService } from "./config.service";
|
||||
import { ConfigService } from './config.service';
|
||||
|
||||
let configSvc: ConfigService | undefined;
|
||||
|
||||
|
||||
@ -5,28 +5,28 @@ export interface IApiError {
|
||||
}
|
||||
|
||||
export const errorSchema = {
|
||||
type: "object",
|
||||
type: 'object',
|
||||
properties: {
|
||||
code: {
|
||||
type: "number",
|
||||
description: "HTTP error code",
|
||||
type: 'number',
|
||||
description: 'HTTP error code',
|
||||
},
|
||||
name: {
|
||||
type: "string",
|
||||
description: "Exception class name",
|
||||
type: 'string',
|
||||
description: 'Exception class name',
|
||||
},
|
||||
message: {
|
||||
type: "string",
|
||||
description: "Exception message",
|
||||
type: 'string',
|
||||
description: 'Exception message',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const errorResponseSchema = {
|
||||
type: "object",
|
||||
type: 'object',
|
||||
properties: {
|
||||
data: {
|
||||
type: "object",
|
||||
type: 'object',
|
||||
nullable: true,
|
||||
},
|
||||
error: errorSchema,
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
import { FastifyReply, FastifyRequest } from "fastify";
|
||||
import { NotHtmlMimetypeError, TxtDotError } from "./main";
|
||||
import { getFastifyError } from "./validation";
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { NotHtmlMimetypeError, TxtDotError } from './main';
|
||||
import { getFastifyError } from './validation';
|
||||
|
||||
import { IGetSchema } from "../types/requests/browser";
|
||||
import getConfig from "../config/main";
|
||||
import { IGetSchema } from '../types/requests/browser';
|
||||
import getConfig from '../config/main';
|
||||
|
||||
export default function errorHandler(
|
||||
error: Error,
|
||||
req: FastifyRequest,
|
||||
reply: FastifyReply
|
||||
) {
|
||||
if (req.originalUrl.startsWith("/api/")) {
|
||||
if (req.originalUrl.startsWith('/api/')) {
|
||||
return apiErrorHandler(error, reply);
|
||||
}
|
||||
|
||||
@ -43,26 +43,23 @@ function apiErrorHandler(error: Error, reply: FastifyReply) {
|
||||
|
||||
function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) {
|
||||
if (getFastifyError(error)?.statusCode === 400) {
|
||||
return reply.code(400).view("/templates/error.ejs", {
|
||||
return reply.code(400).view('/templates/error.ejs', {
|
||||
url,
|
||||
code: 400,
|
||||
description: `Invalid parameter specified: ${error.message}`,
|
||||
})
|
||||
}
|
||||
|
||||
if (error instanceof TxtDotError) {
|
||||
return reply.code(error.code).view("/templates/error.ejs", {
|
||||
url,
|
||||
code: error.code,
|
||||
description: error.description,
|
||||
proxyBtn: (
|
||||
error instanceof NotHtmlMimetypeError &&
|
||||
getConfig().proxy_res
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
return reply.code(500).view("/templates/error.ejs", {
|
||||
if (error instanceof TxtDotError) {
|
||||
return reply.code(error.code).view('/templates/error.ejs', {
|
||||
url,
|
||||
code: error.code,
|
||||
description: error.description,
|
||||
proxyBtn: error instanceof NotHtmlMimetypeError && getConfig().proxy_res,
|
||||
});
|
||||
}
|
||||
|
||||
return reply.code(500).view('/templates/error.ejs', {
|
||||
url,
|
||||
code: 500,
|
||||
description: `${error.name}: ${error.message}`,
|
||||
|
||||
@ -1,15 +1,11 @@
|
||||
import getConfig from "../config/main";
|
||||
import getConfig from '../config/main';
|
||||
|
||||
export abstract class TxtDotError extends Error {
|
||||
code: number;
|
||||
name: string;
|
||||
description: string;
|
||||
|
||||
constructor(
|
||||
code: number,
|
||||
name: string,
|
||||
description: string,
|
||||
) {
|
||||
constructor(code: number, name: string, description: string) {
|
||||
super(description);
|
||||
this.code = code;
|
||||
this.name = name;
|
||||
@ -19,21 +15,13 @@ export abstract class TxtDotError extends Error {
|
||||
|
||||
export class EngineParseError extends TxtDotError {
|
||||
constructor(message: string) {
|
||||
super(
|
||||
422,
|
||||
"EngineParseError",
|
||||
`Parse error: ${message}`,
|
||||
);
|
||||
super(422, 'EngineParseError', `Parse error: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
export class LocalResourceError extends TxtDotError {
|
||||
constructor() {
|
||||
super(
|
||||
403,
|
||||
"LocalResourceError",
|
||||
"Proxying local resources is forbidden.",
|
||||
);
|
||||
super(403, 'LocalResourceError', 'Proxying local resources is forbidden.');
|
||||
}
|
||||
}
|
||||
|
||||
@ -41,12 +29,11 @@ export class NotHtmlMimetypeError extends TxtDotError {
|
||||
constructor() {
|
||||
super(
|
||||
421,
|
||||
"NotHtmlMimetypeError",
|
||||
"Received non-HTML content, " + (
|
||||
getConfig().proxy_res ?
|
||||
"use proxy instead of parser." :
|
||||
"proxying is disabled by the instance admin."
|
||||
),
|
||||
'NotHtmlMimetypeError',
|
||||
'Received non-HTML content, ' +
|
||||
(getConfig().proxy_res
|
||||
? 'use proxy instead of parser.'
|
||||
: 'proxying is disabled by the instance admin.')
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { HandlerInput } from "./handler-input";
|
||||
import { IHandlerOutput } from "./handler.interface";
|
||||
import { EngineParseError } from "../errors/main";
|
||||
import { HandlerInput } from './handler-input';
|
||||
import { IHandlerOutput } from './handler.interface';
|
||||
import { EngineParseError } from '../errors/main';
|
||||
|
||||
export default async function google(
|
||||
input: HandlerInput
|
||||
@ -8,18 +8,18 @@ export default async function google(
|
||||
const window = input.parseDom().window;
|
||||
|
||||
const googleAnchors = [
|
||||
...window.document.querySelectorAll("a[jsname=UWckNb]"),
|
||||
...window.document.querySelectorAll('a[jsname=UWckNb]'),
|
||||
] as HTMLAnchorElement[];
|
||||
|
||||
if (!googleAnchors) {
|
||||
throw new EngineParseError(
|
||||
"Failed to find anchors in search result [google]"
|
||||
'Failed to find anchors in search result [google]'
|
||||
);
|
||||
}
|
||||
|
||||
const results = googleAnchors
|
||||
.map((a: HTMLAnchorElement): GoogleProps => {
|
||||
const parsedHref = new URL(new URL(a.href).searchParams.get("url")!);
|
||||
const parsedHref = new URL(new URL(a.href).searchParams.get('url')!);
|
||||
return {
|
||||
href: a.href!,
|
||||
siteName: parsedHref.hostname,
|
||||
@ -43,7 +43,7 @@ export default async function google(
|
||||
});
|
||||
|
||||
const search = window.document.getElementById(
|
||||
"APjFqb"
|
||||
'APjFqb'
|
||||
) as HTMLTextAreaElement;
|
||||
|
||||
const searchForm = `
|
||||
@ -54,18 +54,18 @@ export default async function google(
|
||||
`;
|
||||
|
||||
return {
|
||||
content: `${searchForm}${content.join("")}`,
|
||||
textContent: textContent.join("\n"),
|
||||
content: `${searchForm}${content.join('')}`,
|
||||
textContent: textContent.join('\n'),
|
||||
};
|
||||
}
|
||||
|
||||
export const GoogleDomains = [
|
||||
"google.*",
|
||||
"google.co.*",
|
||||
"google.com.*",
|
||||
"www.google.*",
|
||||
"www.google.co.*",
|
||||
"www.google.com.*",
|
||||
'google.*',
|
||||
'google.co.*',
|
||||
'google.com.*',
|
||||
'www.google.*',
|
||||
'www.google.co.*',
|
||||
'www.google.com.*',
|
||||
];
|
||||
|
||||
interface GoogleProps {
|
||||
|
||||
@ -1,14 +1,11 @@
|
||||
import { JSDOM } from "jsdom";
|
||||
import { parseHTML } from 'linkedom';
|
||||
|
||||
export class HandlerInput {
|
||||
private data: string;
|
||||
private url: string;
|
||||
private dom?: JSDOM;
|
||||
private dom?: Window;
|
||||
|
||||
constructor(
|
||||
data: string,
|
||||
url: string,
|
||||
) {
|
||||
constructor(data: string, url: string) {
|
||||
this.data = data;
|
||||
this.url = url;
|
||||
}
|
||||
@ -17,12 +14,12 @@ export class HandlerInput {
|
||||
return this.url;
|
||||
}
|
||||
|
||||
parseDom(): JSDOM {
|
||||
parseDom(): Window {
|
||||
if (this.dom) {
|
||||
return this.dom;
|
||||
}
|
||||
|
||||
this.dom = new JSDOM(this.data, { url: this.url });
|
||||
this.dom = parseHTML(this.data);
|
||||
return this.dom;
|
||||
}
|
||||
}
|
||||
|
||||
@ -6,19 +6,19 @@ export interface IHandlerOutput {
|
||||
}
|
||||
|
||||
export const handlerSchema = {
|
||||
type: "object",
|
||||
type: 'object',
|
||||
properties: {
|
||||
content: {
|
||||
type: "string",
|
||||
type: 'string',
|
||||
},
|
||||
textContent: {
|
||||
type: "string",
|
||||
type: 'string',
|
||||
},
|
||||
title: {
|
||||
type: "string",
|
||||
type: 'string',
|
||||
},
|
||||
lang: {
|
||||
type: "string",
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@ -1,42 +1,42 @@
|
||||
import { IHandlerOutput } from "./handler.interface";
|
||||
import { Engines, EngineFunction, EnginesMatch } from "../types/handlers";
|
||||
import axios from "../types/axios";
|
||||
import { IHandlerOutput } from './handler.interface';
|
||||
import { Engines, EngineFunction, EnginesMatch } from '../types/handlers';
|
||||
import axios from '../types/axios';
|
||||
|
||||
import micromatch from "micromatch";
|
||||
import micromatch from 'micromatch';
|
||||
|
||||
import { JSDOM } from "jsdom";
|
||||
import DOMPurify from "dompurify";
|
||||
import DOMPurify from 'dompurify';
|
||||
|
||||
import { Readable } from "stream";
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import readability from "./readability";
|
||||
import google, { GoogleDomains } from "./google";
|
||||
import stackoverflow, { StackOverflowDomains } from "./stackoverflow/main";
|
||||
import readability from './readability';
|
||||
import google, { GoogleDomains } from './google';
|
||||
import stackoverflow, { StackOverflowDomains } from './stackoverflow/main';
|
||||
|
||||
import isLocalResource from "../utils/islocal";
|
||||
import isLocalResource from '../utils/islocal';
|
||||
|
||||
import { LocalResourceError, NotHtmlMimetypeError } from "../errors/main";
|
||||
import { HandlerInput } from "./handler-input";
|
||||
import { decodeStream, parseEncodingName } from "../utils/http";
|
||||
import replaceHref from "../utils/replace-href";
|
||||
import { LocalResourceError, NotHtmlMimetypeError } from '../errors/main';
|
||||
import { HandlerInput } from './handler-input';
|
||||
import { decodeStream, parseEncodingName } from '../utils/http';
|
||||
import replaceHref from '../utils/replace-href';
|
||||
import { parseHTML } from 'linkedom';
|
||||
|
||||
export default async function handlePage(
|
||||
url: string, // remote URL
|
||||
remoteUrl: string, // remote URL
|
||||
requestUrl: URL, // proxy URL
|
||||
engine?: string,
|
||||
redirectPath: string = "get",
|
||||
redirectPath: string = 'get'
|
||||
): Promise<IHandlerOutput> {
|
||||
const urlObj = new URL(url);
|
||||
const urlObj = new URL(remoteUrl);
|
||||
|
||||
if (await isLocalResource(urlObj)) {
|
||||
throw new LocalResourceError();
|
||||
}
|
||||
|
||||
const response = await axios.get(url);
|
||||
const response = await axios.get(remoteUrl);
|
||||
const data: Readable = response.data;
|
||||
const mime: string | undefined = response.headers["content-type"]?.toString();
|
||||
const mime: string | undefined = response.headers['content-type']?.toString();
|
||||
|
||||
if (mime && mime.indexOf("text/html") === -1) {
|
||||
if (mime && mime.indexOf('text/html') === -1) {
|
||||
throw new NotHtmlMimetypeError();
|
||||
}
|
||||
|
||||
@ -44,17 +44,17 @@ export default async function handlePage(
|
||||
const output = await handler(
|
||||
new HandlerInput(
|
||||
await decodeStream(data, parseEncodingName(mime)),
|
||||
url,
|
||||
remoteUrl
|
||||
)
|
||||
);
|
||||
|
||||
// post-process
|
||||
|
||||
const dom = new JSDOM(output.content, { url });
|
||||
replaceHref(dom, requestUrl, engine, redirectPath);
|
||||
const dom = parseHTML(output.content);
|
||||
replaceHref(dom, requestUrl, new URL(remoteUrl), engine, redirectPath);
|
||||
|
||||
const purify = DOMPurify(dom.window);
|
||||
output.content = purify.sanitize(dom.serialize());
|
||||
output.content = purify.sanitize(dom.document.toString());
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import { HandlerInput } from "./handler-input";
|
||||
import { IHandlerOutput } from "./handler.interface";
|
||||
import { EngineParseError } from "../errors/main";
|
||||
import { Readability } from '@mozilla/readability';
|
||||
import { HandlerInput } from './handler-input';
|
||||
import { IHandlerOutput } from './handler.interface';
|
||||
import { EngineParseError } from '../errors/main';
|
||||
|
||||
export default async function readability(
|
||||
input: HandlerInput,
|
||||
input: HandlerInput
|
||||
): Promise<IHandlerOutput> {
|
||||
const reader = new Readability(input.parseDom().window.document);
|
||||
const parsed = reader.parse();
|
||||
|
||||
if (!parsed) {
|
||||
throw new EngineParseError("Failed to parse [readability]");
|
||||
throw new EngineParseError('Failed to parse [readability]');
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@ -1,30 +1,30 @@
|
||||
import { HandlerInput } from "../handler-input";
|
||||
import { IHandlerOutput } from "../handler.interface";
|
||||
import { EngineParseError } from "../../errors/main";
|
||||
import qPostsHandler from "./questions-posts";
|
||||
import { HandlerInput } from '../handler-input';
|
||||
import { IHandlerOutput } from '../handler.interface';
|
||||
import { EngineParseError } from '../../errors/main';
|
||||
import qPostsHandler from './questions-posts';
|
||||
|
||||
export default async function stackoverflow(
|
||||
input: HandlerInput,
|
||||
input: HandlerInput
|
||||
): Promise<IHandlerOutput> {
|
||||
const window = input.parseDom().window;
|
||||
|
||||
const url = new URL(window.location.href);
|
||||
const path = url.pathname.split("/").filter((p) => p !== "");
|
||||
const path = url.pathname.split('/').filter((p) => p !== '');
|
||||
|
||||
let result: IHandlerOutput = {
|
||||
content: "",
|
||||
textContent: "",
|
||||
title: "",
|
||||
lang: "",
|
||||
content: '',
|
||||
textContent: '',
|
||||
title: '',
|
||||
lang: '',
|
||||
};
|
||||
|
||||
if (path[0] === "questions") {
|
||||
if (path[0] === 'questions') {
|
||||
if (path.length === 3) {
|
||||
result = await qPostsHandler(window);
|
||||
} else if (path.length === 1) {
|
||||
result.content = "questions";
|
||||
result.content = 'questions';
|
||||
} else {
|
||||
throw new EngineParseError("Invalid URL [stackoverflow]");
|
||||
throw new EngineParseError('Invalid URL [stackoverflow]');
|
||||
}
|
||||
}
|
||||
|
||||
@ -32,12 +32,12 @@ export default async function stackoverflow(
|
||||
}
|
||||
|
||||
export const StackOverflowDomains = [
|
||||
"stackoverflow.com",
|
||||
"*.stackoverflow.com",
|
||||
"*.stackexchange.com",
|
||||
"askubuntu.com",
|
||||
"stackapps.com",
|
||||
"mathoverflow.net",
|
||||
"superuser.com",
|
||||
"serverfault.com",
|
||||
'stackoverflow.com',
|
||||
'*.stackoverflow.com',
|
||||
'*.stackexchange.com',
|
||||
'askubuntu.com',
|
||||
'stackapps.com',
|
||||
'mathoverflow.net',
|
||||
'superuser.com',
|
||||
'serverfault.com',
|
||||
];
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
export default function postParser(el: Element | null): string {
|
||||
if (!el) {
|
||||
return "";
|
||||
return '';
|
||||
}
|
||||
const body = el.querySelector(".js-post-body")?.innerHTML || "";
|
||||
const voteCount = el.querySelector(".js-vote-count")?.textContent || "";
|
||||
const body = el.querySelector('.js-post-body')?.innerHTML || '';
|
||||
const voteCount = el.querySelector('.js-vote-count')?.textContent || '';
|
||||
|
||||
return `<h3>${voteCount} votes</h3>${body}`;
|
||||
}
|
||||
|
||||
@ -1,26 +1,25 @@
|
||||
import { DOMWindow } from "jsdom";
|
||||
import { IHandlerOutput } from "../handler.interface";
|
||||
import postParser from "./post-parser";
|
||||
import { IHandlerOutput } from '../handler.interface';
|
||||
import postParser from './post-parser';
|
||||
|
||||
export default async function qPostsHandler(
|
||||
window: DOMWindow
|
||||
window: Window
|
||||
): Promise<IHandlerOutput> {
|
||||
const questionEl = window.document.getElementById("question");
|
||||
const questionEl = window.document.getElementById('question');
|
||||
const question = postParser(questionEl);
|
||||
|
||||
const title =
|
||||
window.document.querySelector(".question-hyperlink")?.innerHTML || "";
|
||||
window.document.querySelector('.question-hyperlink')?.innerHTML || '';
|
||||
|
||||
const allAnswers = [...window.document.querySelectorAll(".answer")];
|
||||
const allAnswers = [...window.document.querySelectorAll('.answer')];
|
||||
|
||||
const answers = allAnswers.map((a) => postParser(a));
|
||||
|
||||
return {
|
||||
content: `${question}<hr>${answers.length} answers <hr>${answers.join(
|
||||
"<hr>"
|
||||
'<hr>'
|
||||
)}`,
|
||||
textContent: "question",
|
||||
textContent: 'question',
|
||||
title,
|
||||
lang: "en",
|
||||
lang: 'en',
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
export default {
|
||||
version: "1.4.0",
|
||||
version: '1.4.0',
|
||||
description:
|
||||
"txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts",
|
||||
'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts',
|
||||
};
|
||||
|
||||
@ -1,13 +1,17 @@
|
||||
import { FastifyInstance } from "fastify";
|
||||
import { FastifyInstance } from 'fastify';
|
||||
|
||||
import { EngineRequest, IParseSchema, parseSchema } from "../../types/requests/api";
|
||||
import {
|
||||
EngineRequest,
|
||||
IParseSchema,
|
||||
parseSchema,
|
||||
} from '../../types/requests/api';
|
||||
|
||||
import handlePage from "../../handlers/main";
|
||||
import { generateRequestUrl } from "../../utils/generate";
|
||||
import handlePage from '../../handlers/main';
|
||||
import { generateRequestUrl } from '../../utils/generate';
|
||||
|
||||
export default async function parseRoute(fastify: FastifyInstance) {
|
||||
fastify.get<IParseSchema>(
|
||||
"/api/parse",
|
||||
'/api/parse',
|
||||
{ schema: parseSchema },
|
||||
async (request: EngineRequest) => {
|
||||
return {
|
||||
|
||||
@ -1,16 +1,16 @@
|
||||
import { FastifyInstance } from "fastify";
|
||||
import { FastifyInstance } from 'fastify';
|
||||
|
||||
import { IParseSchema, rawHtmlSchema } from "../../types/requests/api";
|
||||
import { IParseSchema, rawHtmlSchema } from '../../types/requests/api';
|
||||
|
||||
import handlePage from "../../handlers/main";
|
||||
import { generateRequestUrl } from "../../utils/generate";
|
||||
import handlePage from '../../handlers/main';
|
||||
import { generateRequestUrl } from '../../utils/generate';
|
||||
|
||||
export default async function rawHtml(fastify: FastifyInstance) {
|
||||
fastify.get<IParseSchema>(
|
||||
"/api/raw-html",
|
||||
'/api/raw-html',
|
||||
{ schema: rawHtmlSchema },
|
||||
async (request, reply) => {
|
||||
reply.type("text/html; charset=utf-8");
|
||||
reply.type('text/html; charset=utf-8');
|
||||
return (
|
||||
await handlePage(
|
||||
request.query.url,
|
||||
@ -20,7 +20,7 @@ export default async function rawHtml(fastify: FastifyInstance) {
|
||||
request.originalUrl
|
||||
),
|
||||
request.query.engine,
|
||||
"api/raw-html"
|
||||
'api/raw-html'
|
||||
)
|
||||
).content;
|
||||
}
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
import { FastifyInstance } from "fastify";
|
||||
import { FastifyInstance } from 'fastify';
|
||||
|
||||
import { GetSchema, IGetSchema } from "../../types/requests/browser";
|
||||
import handlePage from "../../handlers/main";
|
||||
import { generateRequestUrl } from "../../utils/generate";
|
||||
import { GetSchema, IGetSchema } from '../../types/requests/browser';
|
||||
import handlePage from '../../handlers/main';
|
||||
import { generateRequestUrl } from '../../utils/generate';
|
||||
|
||||
export default async function getRoute(fastify: FastifyInstance) {
|
||||
fastify.get<IGetSchema>(
|
||||
"/get",
|
||||
'/get',
|
||||
{ schema: GetSchema },
|
||||
async (request, reply) => {
|
||||
const remoteUrl = request.query.url;
|
||||
@ -22,12 +22,12 @@ export default async function getRoute(fastify: FastifyInstance) {
|
||||
engine
|
||||
);
|
||||
|
||||
if (request.query.format === "text") {
|
||||
reply.type("text/plain; charset=utf-8");
|
||||
if (request.query.format === 'text') {
|
||||
reply.type('text/plain; charset=utf-8');
|
||||
return parsed.textContent;
|
||||
} else {
|
||||
reply.type("text/html; charset=utf-8");
|
||||
return reply.view("/templates/get.ejs", { parsed, remoteUrl });
|
||||
reply.type('text/html; charset=utf-8');
|
||||
return reply.view('/templates/get.ejs', { parsed, remoteUrl });
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
import { FastifyInstance } from "fastify";
|
||||
import { FastifyInstance } from 'fastify';
|
||||
|
||||
import publicConfig from "../../publicConfig";
|
||||
import { engineList } from "../../handlers/main";
|
||||
import { indexSchema } from "../../types/requests/browser";
|
||||
import publicConfig from '../../publicConfig';
|
||||
import { engineList } from '../../handlers/main';
|
||||
import { indexSchema } from '../../types/requests/browser';
|
||||
|
||||
export default async function indexRoute(fastify: FastifyInstance) {
|
||||
fastify.get("/", { schema: indexSchema }, async (_, reply) => {
|
||||
return reply.view("/templates/index.ejs", { publicConfig, engineList });
|
||||
fastify.get('/', { schema: indexSchema }, async (_, reply) => {
|
||||
return reply.view('/templates/index.ejs', { publicConfig, engineList });
|
||||
});
|
||||
}
|
||||
|
||||
@ -1,17 +1,19 @@
|
||||
import { FastifyInstance } from "fastify";
|
||||
import { IProxySchema, ProxySchema } from "../../types/requests/browser";
|
||||
import axios from "../../types/axios";
|
||||
import { FastifyInstance } from 'fastify';
|
||||
import { IProxySchema, ProxySchema } from '../../types/requests/browser';
|
||||
import axios from '../../types/axios';
|
||||
|
||||
export default async function proxyRoute(fastify: FastifyInstance) {
|
||||
fastify.get<IProxySchema>(
|
||||
"/proxy",
|
||||
'/proxy',
|
||||
{ schema: ProxySchema },
|
||||
async (request, reply) => {
|
||||
const response = await axios.get(request.query.url);
|
||||
const mime: string | undefined = response.headers["content-type"]?.toString();
|
||||
const clen: string | undefined = response.headers["content-length"]?.toString();
|
||||
mime && reply.header("Content-Type", mime);
|
||||
clen && reply.header("Content-Length", Number(clen));
|
||||
const mime: string | undefined =
|
||||
response.headers['content-type']?.toString();
|
||||
const clen: string | undefined =
|
||||
response.headers['content-length']?.toString();
|
||||
mime && reply.header('Content-Type', mime);
|
||||
clen && reply.header('Content-Length', Number(clen));
|
||||
return reply.send(response.data);
|
||||
}
|
||||
);
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
import axios from "axios";
|
||||
import axios from 'axios';
|
||||
|
||||
export default axios.create({
|
||||
headers: {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
|
||||
'User-Agent':
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0',
|
||||
},
|
||||
responseType: "stream",
|
||||
responseType: 'stream',
|
||||
});
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { HandlerInput } from "../handlers/handler-input";
|
||||
import { IHandlerOutput } from "../handlers/handler.interface";
|
||||
import { HandlerInput } from '../handlers/handler-input';
|
||||
import { IHandlerOutput } from '../handlers/handler.interface';
|
||||
|
||||
export interface Engines {
|
||||
[key: string]: EngineFunction;
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
import { FastifySchema, FastifyRequest } from "fastify";
|
||||
import { IApiError, errorResponseSchema } from "../../errors/api";
|
||||
import { handlerSchema } from "../../handlers/handler.interface";
|
||||
import { engineList } from "../../handlers/main";
|
||||
import { FromSchema } from "json-schema-to-ts";
|
||||
import { FastifySchema, FastifyRequest } from 'fastify';
|
||||
import { IApiError, errorResponseSchema } from '../../errors/api';
|
||||
import { handlerSchema } from '../../handlers/handler.interface';
|
||||
import { engineList } from '../../handlers/main';
|
||||
import { FromSchema } from 'json-schema-to-ts';
|
||||
|
||||
export interface IApiResponse<T> {
|
||||
data?: T;
|
||||
@ -10,38 +10,38 @@ export interface IApiResponse<T> {
|
||||
}
|
||||
|
||||
export const parseQuerySchema = {
|
||||
type: "object",
|
||||
required: ["url"],
|
||||
type: 'object',
|
||||
required: ['url'],
|
||||
properties: {
|
||||
url: {
|
||||
type: "string",
|
||||
description: "URL",
|
||||
type: 'string',
|
||||
description: 'URL',
|
||||
},
|
||||
engine: {
|
||||
type: "string",
|
||||
enum: [...engineList, ""],
|
||||
type: 'string',
|
||||
enum: [...engineList, ''],
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
export const parseSchema: FastifySchema = {
|
||||
description: "Parse the page and get all data from the engine",
|
||||
description: 'Parse the page and get all data from the engine',
|
||||
querystring: parseQuerySchema,
|
||||
response: {
|
||||
"2xx": {
|
||||
type: "object",
|
||||
'2xx': {
|
||||
type: 'object',
|
||||
properties: {
|
||||
data: handlerSchema,
|
||||
error: {
|
||||
type: "object",
|
||||
type: 'object',
|
||||
nullable: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
"4xx": errorResponseSchema,
|
||||
"5xx": errorResponseSchema,
|
||||
'4xx': errorResponseSchema,
|
||||
'5xx': errorResponseSchema,
|
||||
},
|
||||
produces: ["text/json"],
|
||||
produces: ['text/json'],
|
||||
};
|
||||
|
||||
export interface IParseSchema {
|
||||
@ -49,9 +49,9 @@ export interface IParseSchema {
|
||||
}
|
||||
|
||||
export const rawHtmlSchema: FastifySchema = {
|
||||
description: "Parse the page and get raw HTML from the engine",
|
||||
description: 'Parse the page and get raw HTML from the engine',
|
||||
querystring: parseQuerySchema,
|
||||
produces: ["text/html"],
|
||||
produces: ['text/html'],
|
||||
};
|
||||
|
||||
export type EngineRequest = FastifyRequest<{
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { FastifySchema } from "fastify";
|
||||
import { engineList } from "../../handlers/main";
|
||||
import { FromSchema } from "json-schema-to-ts";
|
||||
import { FastifySchema } from 'fastify';
|
||||
import { engineList } from '../../handlers/main';
|
||||
import { FromSchema } from 'json-schema-to-ts';
|
||||
|
||||
export interface IGetSchema {
|
||||
Querystring: IGetQuerySchema;
|
||||
@ -11,52 +11,52 @@ export interface IProxySchema {
|
||||
}
|
||||
|
||||
export const getQuerySchema = {
|
||||
type: "object",
|
||||
required: ["url"],
|
||||
type: 'object',
|
||||
required: ['url'],
|
||||
properties: {
|
||||
url: {
|
||||
type: "string",
|
||||
description: "URL",
|
||||
type: 'string',
|
||||
description: 'URL',
|
||||
},
|
||||
format: {
|
||||
type: "string",
|
||||
enum: ["text", "html", ""],
|
||||
default: "html",
|
||||
type: 'string',
|
||||
enum: ['text', 'html', ''],
|
||||
default: 'html',
|
||||
},
|
||||
engine: {
|
||||
type: "string",
|
||||
enum: [...engineList, ""],
|
||||
type: 'string',
|
||||
enum: [...engineList, ''],
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
export type IGetQuerySchema = FromSchema<typeof getQuerySchema>;
|
||||
|
||||
export const proxyQuerySchema = {
|
||||
type: "object",
|
||||
required: ["url"],
|
||||
type: 'object',
|
||||
required: ['url'],
|
||||
properties: {
|
||||
url: {
|
||||
type: "string",
|
||||
description: "URL",
|
||||
type: 'string',
|
||||
description: 'URL',
|
||||
},
|
||||
}
|
||||
},
|
||||
} as const;
|
||||
export type IProxyQuerySchema = FromSchema<typeof proxyQuerySchema>;
|
||||
|
||||
export const indexSchema = {
|
||||
hide: true,
|
||||
produces: ["text/html"],
|
||||
produces: ['text/html'],
|
||||
};
|
||||
|
||||
export const GetSchema: FastifySchema = {
|
||||
description: "Get page",
|
||||
description: 'Get page',
|
||||
hide: true,
|
||||
querystring: getQuerySchema,
|
||||
produces: ["text/html", "text/plain"],
|
||||
produces: ['text/html', 'text/plain'],
|
||||
};
|
||||
|
||||
export const ProxySchema: FastifySchema = {
|
||||
description: "Proxy resource",
|
||||
description: 'Proxy resource',
|
||||
hide: true,
|
||||
querystring: proxyQuerySchema,
|
||||
}
|
||||
};
|
||||
|
||||
@ -8,25 +8,35 @@ export function generateRequestUrl(
|
||||
|
||||
export function generateParserUrl(
|
||||
requestUrl: URL,
|
||||
remoteUrl: URL,
|
||||
href: string,
|
||||
engine?: string,
|
||||
redirect_url: string = "get"
|
||||
redirect_url: string = 'get'
|
||||
): string {
|
||||
const parsedHref = new URL(href);
|
||||
const realURL = getRealURL(href, remoteUrl);
|
||||
|
||||
const hash = parsedHref.hash; // save #hash
|
||||
parsedHref.hash = ""; // remove
|
||||
const hash = realURL.hash; // save #hash
|
||||
realURL.hash = ''; // remove
|
||||
|
||||
const urlParam = `?url=${encodeURIComponent(parsedHref.toString())}`;
|
||||
const engineParam = engine ? `&engine=${engine}` : "";
|
||||
const urlParam = `?url=${encodeURIComponent(realURL.toString())}`;
|
||||
const engineParam = engine ? `&engine=${engine}` : '';
|
||||
|
||||
return `${requestUrl.origin}/${redirect_url}${urlParam}${engineParam}${hash}`;
|
||||
}
|
||||
|
||||
export function generateProxyUrl(
|
||||
requestUrl: URL,
|
||||
href: string,
|
||||
remoteUrl: URL,
|
||||
href: string
|
||||
): string {
|
||||
const urlParam = `?url=${encodeURIComponent(href)}`;
|
||||
const realHref = getRealURL(href, remoteUrl);
|
||||
|
||||
const urlParam = `?url=${encodeURIComponent(realHref.href)}`;
|
||||
return `${requestUrl.origin}/proxy${urlParam}`;
|
||||
}
|
||||
|
||||
function getRealURL(href: string, remoteUrl: URL) {
|
||||
return href.startsWith('http')
|
||||
? new URL(href)
|
||||
: new URL(href, remoteUrl.href);
|
||||
}
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
import { Readable } from "stream";
|
||||
import iconv from "iconv-lite";
|
||||
import { Readable } from 'stream';
|
||||
import iconv from 'iconv-lite';
|
||||
|
||||
export async function decodeStream(
|
||||
data: Readable,
|
||||
charset: string = "utf-8",
|
||||
charset: string = 'utf-8'
|
||||
): Promise<string> {
|
||||
const strm = data.pipe(iconv.decodeStream(charset)) as IconvStream;
|
||||
return await new Promise(resolve => {
|
||||
return await new Promise((resolve) => {
|
||||
strm.collect((_err: Error, body: string) => {
|
||||
resolve(body);
|
||||
});
|
||||
@ -16,7 +16,7 @@ export async function decodeStream(
|
||||
export function parseEncodingName(ctype?: string): string {
|
||||
const match = ctype?.match(/charset=([A-Za-z0-9-]+)$/);
|
||||
if (!match) {
|
||||
return "utf-8";
|
||||
return 'utf-8';
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
@ -1,45 +1,43 @@
|
||||
import dns from "dns";
|
||||
import ipRangeCheck from "ip-range-check";
|
||||
import dns from 'dns';
|
||||
import ipRangeCheck from 'ip-range-check';
|
||||
|
||||
const subnets = [
|
||||
"0.0.0.0/8",
|
||||
"127.0.0.0/8",
|
||||
"10.0.0.0/8",
|
||||
"100.64.0.0/10",
|
||||
"169.254.0.0/16",
|
||||
"172.16.0.0/12",
|
||||
"192.0.0.0/24",
|
||||
"192.0.2.0/24",
|
||||
"192.88.99.0/24",
|
||||
"192.168.0.0/16",
|
||||
"198.18.0.0/15",
|
||||
"198.51.100.0/24",
|
||||
"203.0.113.0/24",
|
||||
"224.0.0.0/4",
|
||||
"233.252.0.0/24",
|
||||
"240.0.0.0/4",
|
||||
"255.255.255.255/32",
|
||||
"::/128",
|
||||
"::1/128",
|
||||
"::ffff:0:0/96",
|
||||
"::ffff:0:0:0/96",
|
||||
"64:ff9b::/96",
|
||||
"64:ff9b:1::/48",
|
||||
"100::/64",
|
||||
"2001:0000::/32",
|
||||
"2001:20::/28",
|
||||
"2001:db8::/32",
|
||||
"2002::/16",
|
||||
"fc00::/7",
|
||||
"fe80::/64",
|
||||
"ff00::/8",
|
||||
'0.0.0.0/8',
|
||||
'127.0.0.0/8',
|
||||
'10.0.0.0/8',
|
||||
'100.64.0.0/10',
|
||||
'169.254.0.0/16',
|
||||
'172.16.0.0/12',
|
||||
'192.0.0.0/24',
|
||||
'192.0.2.0/24',
|
||||
'192.88.99.0/24',
|
||||
'192.168.0.0/16',
|
||||
'198.18.0.0/15',
|
||||
'198.51.100.0/24',
|
||||
'203.0.113.0/24',
|
||||
'224.0.0.0/4',
|
||||
'233.252.0.0/24',
|
||||
'240.0.0.0/4',
|
||||
'255.255.255.255/32',
|
||||
'::/128',
|
||||
'::1/128',
|
||||
'::ffff:0:0/96',
|
||||
'::ffff:0:0:0/96',
|
||||
'64:ff9b::/96',
|
||||
'64:ff9b:1::/48',
|
||||
'100::/64',
|
||||
'2001:0000::/32',
|
||||
'2001:20::/28',
|
||||
'2001:db8::/32',
|
||||
'2002::/16',
|
||||
'fc00::/7',
|
||||
'fe80::/64',
|
||||
'ff00::/8',
|
||||
];
|
||||
|
||||
export default async function isLocalResource(url: URL): Promise<boolean> {
|
||||
// Resolve domain name
|
||||
const addr = (
|
||||
await dns.promises.lookup(url.hostname)
|
||||
).address;
|
||||
const addr = (await dns.promises.lookup(url.hostname)).address;
|
||||
|
||||
// Check if IP is in local network
|
||||
return ipRangeCheck(addr, subnets);
|
||||
|
||||
@ -1,72 +1,53 @@
|
||||
import { JSDOM } from "jsdom";
|
||||
import { generateParserUrl, generateProxyUrl } from "./generate";
|
||||
import getConfig from "../config/main";
|
||||
import { generateParserUrl, generateProxyUrl } from './generate';
|
||||
import getConfig from '../config/main';
|
||||
|
||||
export default function replaceHref(
|
||||
dom: JSDOM,
|
||||
dom: Window,
|
||||
requestUrl: URL,
|
||||
remoteUrl: URL,
|
||||
engine?: string,
|
||||
redirectPath: string = "get",
|
||||
redirectPath: string = 'get'
|
||||
) {
|
||||
const doc = dom.window.document;
|
||||
|
||||
const doc: Document = dom.window.document;
|
||||
const parserUrl = (href: string) =>
|
||||
href.startsWith("http") ? generateParserUrl(
|
||||
requestUrl,
|
||||
href,
|
||||
engine,
|
||||
redirectPath,
|
||||
) : href;
|
||||
const proxyUrl = (href: string) =>
|
||||
href.startsWith("http") ? generateProxyUrl(
|
||||
requestUrl,
|
||||
href,
|
||||
) : href;
|
||||
generateParserUrl(requestUrl, remoteUrl, href, engine, redirectPath);
|
||||
|
||||
modifyLinks(
|
||||
doc.getElementsByTagName("a"),
|
||||
"href",
|
||||
parserUrl,
|
||||
);
|
||||
modifyLinks(
|
||||
doc.querySelectorAll("frame,iframe"),
|
||||
"src",
|
||||
parserUrl,
|
||||
);
|
||||
const proxyUrl = (href: string) =>
|
||||
generateProxyUrl(requestUrl, remoteUrl, href);
|
||||
|
||||
modifyLinks(doc.querySelectorAll('a[href]'), 'href', parserUrl);
|
||||
modifyLinks(doc.querySelectorAll('frame,iframe'), 'src', parserUrl);
|
||||
|
||||
if (getConfig().proxy_res) {
|
||||
modifyLinks(
|
||||
doc.querySelectorAll("img,image,video,audio,embed,track,source"),
|
||||
"src",
|
||||
proxyUrl,
|
||||
doc.querySelectorAll('img,image,video,audio,embed,track,source'),
|
||||
'src',
|
||||
proxyUrl
|
||||
);
|
||||
|
||||
modifyLinks(
|
||||
doc.getElementsByTagName("object"),
|
||||
"data",
|
||||
proxyUrl,
|
||||
);
|
||||
|
||||
const sources = doc.querySelectorAll("source,img");
|
||||
modifyLinks(doc.getElementsByTagName('object'), 'data', proxyUrl);
|
||||
const sources = doc.querySelectorAll('source,img');
|
||||
for (const source of sources) {
|
||||
// split srcset by comma
|
||||
// @ts-ignore
|
||||
if (!source.srcset)
|
||||
continue;
|
||||
// @ts-ignore
|
||||
source.srcset = source.srcset.split(",").map(
|
||||
(src: string) => {
|
||||
// @ts-expect-error because I don't know what to do about it.
|
||||
if (!source.srcset) continue;
|
||||
// @ts-expect-error because I don't know what to do about it.
|
||||
source.srcset = source.srcset
|
||||
.split(',')
|
||||
.map((src: string) => {
|
||||
// split src by space
|
||||
const parts = src.trim().split(" ");
|
||||
const parts = src.trim().split(' ');
|
||||
try {
|
||||
// first part is URL
|
||||
// (srcset="http 200w 1x,...")
|
||||
parts[0] = proxyUrl(parts[0]);
|
||||
} catch (_err) { }
|
||||
} catch (_err) {
|
||||
/* empty */
|
||||
}
|
||||
// join by space after splitting
|
||||
return parts.join(" ");
|
||||
}
|
||||
).join(","); // join by comma
|
||||
return parts.join(' ');
|
||||
})
|
||||
.join(','); // join by comma
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -74,12 +55,14 @@ export default function replaceHref(
|
||||
function modifyLinks(
|
||||
nodeList: NodeListOf<Element> | HTMLCollectionOf<Element>,
|
||||
property: string,
|
||||
generateLink: (value: string) => string,
|
||||
generateLink: (value: string) => string
|
||||
) {
|
||||
for (const node of nodeList) {
|
||||
try {
|
||||
// @ts-ignore
|
||||
// @ts-expect-error because I don't know what to do about it.
|
||||
node[property] = generateLink(node[property]);
|
||||
} catch (_err) { }
|
||||
} catch (_err) {
|
||||
/* empty */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -31,7 +31,7 @@ label {
|
||||
|
||||
#url {
|
||||
width: 100%;
|
||||
height: 100%; /* shrink to #submit height */
|
||||
height: 100%; /* shrink to #submit height */
|
||||
|
||||
outline: none;
|
||||
border: 0;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user