feature: allow local resources

Fix: #257

.github/workflows/build-check.yml

@@ -14,7 +14,7 @@ jobs:
 
       - uses: pnpm/action-setup@v4
         with:
-          version: 9
+          version: 10
 
       - name: Install dependencies
         run: pnpm install

Dockerfile

@@ -1,7 +1,10 @@
-FROM node:20-alpine as base
+FROM node:23-alpine AS base
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable
+RUN \
+  npm i -g corepack@latest \
+  corepack enable \
+  corepack use pnpm@latest-10 
 COPY . /app
 WORKDIR /app
 
@@ -12,11 +15,14 @@ FROM base AS build
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
 RUN pnpm run build
 
-FROM node:20-alpine as run
+FROM node:23-alpine AS run
 
 ENV PNPM_HOME="/pnpm"
 ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable
+RUN \
+  npm i -g corepack@latest \
+  corepack enable \
+  corepack use pnpm@latest-10
 WORKDIR /app
 
 COPY --from=prod-deps /app/node_modules /app/node_modules

README.md

@@ -1,3 +1,6 @@
+> [!WARNING]
+> The project is frozen for a while. txtdot needs a complete rewrite in rust and dalet integration, which is not finished yet.
+
 <div align="center">
 
 ![txtdot logo](https://github.com/TxtDot/.github/raw/main/imgs/txtdot.png)

package.json

@@ -7,13 +7,19 @@
     "format:check": "prettier --check ."
   },
   "devDependencies": {
-    "@types/node": "^22.8.0",
-    "@typescript-eslint/eslint-plugin": "^8.11.0",
-    "@typescript-eslint/parser": "^8.11.0",
-    "eslint": "^9.13.0",
-    "lerna": "^8.1.8",
-    "prettier": "^3.3.3",
-    "tsc-watch": "^6.2.0",
-    "typescript": "^5.6.3"
+    "@types/node": "^22.13.4",
+    "@typescript-eslint/eslint-plugin": "^8.24.0",
+    "@typescript-eslint/parser": "^8.24.0",
+    "eslint": "^9.20.1",
+    "lerna": "^8.1.9",
+    "prettier": "^3.5.1",
+    "tsc-watch": "^6.2.1",
+    "typescript": "5.7.2"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "nx",
+      "sharp"
+    ]
   }
 }

packages/plugins/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@txtdot/plugins",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "Official txtdot plugins",
   "main": "dist/lib.js",
   "types": "dist/lib.d.ts",
@@ -21,10 +21,9 @@
     "@mozilla/readability": "^0.5.0",
     "@txtdot/sdk": "workspace:*",
     "html-to-text": "^9.0.5",
-    "linkedom": "^0.18.0"
+    "linkedom": "^0.18.9"
   },
   "devDependencies": {
-    "@types/html-to-text": "^9.0.4",
-    "typescript": "^5.4.5"
+    "@types/html-to-text": "^9.0.4"
   }
 }

packages/plugins/src/engines/searx.tsx

@@ -30,8 +30,8 @@ async function search(
   const articles_parsed = articles.map((a) => {
     const parsed = {
       url:
-        (a.getElementsByClassName('url_wrapper')[0] as HTMLAnchorElement)
-          .href || '',
+        (a.getElementsByClassName('url_header')[0] as HTMLAnchorElement).href ||
+        '',
       title:
         (a.getElementsByTagName('h3')[0] as HTMLHeadingElement).textContent ||
         '',

packages/plugins/tsconfig.json

@@ -1,7 +1,6 @@
 {
   "compilerOptions": {
     /* Visit https://aka.ms/tsconfig to read more about this file */
-
     /* Projects */
     "incremental": true /* Save .tsbuildinfo files to allow for incremental compilation of projects. */,
     // "composite": true /* Enable constraints that allow a TypeScript project to be used with project references. */,

packages/sdk/package.json

@@ -18,11 +18,10 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "linkedom": "^0.18.0",
+    "linkedom": "^0.18.9",
     "route-parser": "^0.0.5"
   },
   "devDependencies": {
-    "@types/route-parser": "^0.1.7",
-    "typescript": "^5.4.5"
+    "@types/route-parser": "^0.1.7"
   }
 }

packages/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@txtdot/server",
-  "version": "1.9.1",
+  "version": "1.10.1",
   "private": true,
   "description": "txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts",
   "main": "dist/app.js",
@@ -18,28 +18,28 @@
     "dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\""
   },
   "dependencies": {
-    "@fastify/one-line-logger": "^2.0.0",
-    "@fastify/static": "^8.0.1",
-    "@fastify/swagger": "^9.1.0",
-    "@fastify/swagger-ui": "^5.0.1",
-    "@fastify/view": "^10.0.1",
+    "@fastify/one-line-logger": "^2.0.2",
+    "@fastify/static": "^8.1.0",
+    "@fastify/swagger": "^9.4.2",
+    "@fastify/swagger-ui": "^5.2.1",
+    "@fastify/view": "^10.0.2",
     "@txtdot/plugins": "workspace:*",
     "@txtdot/sdk": "workspace:*",
-    "axios": "^1.7.7",
-    "dotenv": "^16.3.1",
+    "axios": "^1.7.9",
+    "dotenv": "^16.4.7",
     "ejs": "^3.1.10",
-    "fastify": "^5.0.0",
+    "fastify": "^5.2.1",
     "iconv-lite": "^0.6.3",
     "ip-range-check": "^0.2.0",
-    "isomorphic-dompurify": "^2.16.0",
-    "json-schema-to-ts": "^3.1.0",
-    "linkedom": "^0.18.0",
+    "isomorphic-dompurify": "^2.21.0",
+    "json-schema-to-ts": "^3.1.1",
+    "linkedom": "^0.18.9",
     "micromatch": "^4.0.8",
-    "sharp": "^0.33.3"
+    "sharp": "^0.33.5"
   },
   "devDependencies": {
     "@types/ejs": "^3.1.5",
-    "@types/jsdom": "^21.1.6",
+    "@types/jsdom": "^21.1.7",
     "@types/micromatch": "^4.0.9",
     "clean-css-cli": "^5.6.3",
     "copyfiles": "^2.4.1"

packages/server/src/routes/api/parse.ts

@@ -20,6 +20,7 @@ export default async function parseRoute(fastify: FastifyInstance) {
           generateRequestUrl(
             request.protocol,
             request.hostname,
+            request.port,
             request.originalUrl
           ),
           request.query.engine

packages/server/src/routes/api/raw-html.ts

@@ -17,6 +17,7 @@ export default async function rawHtml(fastify: FastifyInstance) {
           generateRequestUrl(
             request.protocol,
             request.hostname,
+            request.port,
             request.originalUrl
           ),
           request.query.engine,

packages/server/src/routes/browser/get.ts

@@ -18,6 +18,7 @@ export default async function getRoute(fastify: FastifyInstance) {
         generateRequestUrl(
           request.protocol,
           request.hostname,
+          request.port,
           request.originalUrl
         ),
         engine

packages/server/src/types/axios.ts

@@ -1,6 +1,4 @@
 import origAxios, { CreateAxiosDefaults } from 'axios';
-import { isLocalResource, isLocalResourceURL } from '../utils/islocal';
-import { LocalResourceError } from '../errors/main';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const config: CreateAxiosDefaults<any> = {
@@ -14,18 +12,8 @@ const config: CreateAxiosDefaults<any> = {
 const axios = origAxios.create(config);
 
 axios.interceptors.response.use(
-  (response) => {
-    if (isLocalResource(response.request.socket.remoteAddress)) {
-      throw new LocalResourceError();
-    }
-
-    return response;
-  },
-  async (error) => {
-    if (await isLocalResourceURL(new URL(error.config?.url))) {
-      throw new LocalResourceError();
-    }
-
+  (response) => response,
+  (error) => {
     throw error;
   }
 );

packages/server/src/utils/generate.ts

@@ -1,9 +1,12 @@
 export function generateRequestUrl(
   protocol: string,
   host: string,
+  port: number,
   originalUrl: string
 ): URL {
-  return new URL(`${protocol}://${host}${originalUrl}`);
+  return new URL(
+    `${protocol}://${host}${port === null ? '' : `:${port}`}${originalUrl}`
+  );
 }
 
 export function generateParserUrl(