refactor: create monorepo

packages/server/src/app.ts

@@ -0,0 +1,79 @@
+import path from 'path';
+
+import Fastify from 'fastify';
+import fastifyStatic from '@fastify/static';
+import fastifyView from '@fastify/view';
+import fastifySwagger from '@fastify/swagger';
+import fastifySwaggerUi from '@fastify/swagger-ui';
+import ejs from 'ejs';
+
+import indexRoute from './routes/browser/index';
+import getRoute from './routes/browser/get';
+import proxyRoute from './routes/browser/proxy';
+import parseRoute from './routes/api/parse';
+import rawHtml from './routes/api/raw-html';
+
+import errorHandler from './errors/handler';
+import redirectRoute from './routes/browser/redirect';
+
+import configurationRoute from './routes/browser/configuration';
+
+import config from './config';
+
+class App {
+  async init() {
+    const fastify = Fastify({
+      logger: true,
+      trustProxy: config.env.reverse_proxy,
+      connectionTimeout: config.env.timeout,
+    });
+
+    fastify.setErrorHandler(errorHandler);
+
+    fastify.register(fastifyStatic, {
+      root: path.join(process.cwd(), 'static'),
+      prefix: '/static/',
+    });
+
+    fastify.register(fastifyView, {
+      engine: {
+        ejs: ejs,
+      },
+    });
+
+    if (config.env.swagger) {
+      config.dyn.addRoute('/doc');
+      await fastify.register(fastifySwagger, {
+        swagger: {
+          info: {
+            title: 'TXTDot API',
+            description: config.package.description,
+            version: config.package.version,
+          },
+        },
+      });
+      await fastify.register(fastifySwaggerUi, { routePrefix: '/doc' });
+    }
+
+    fastify.addHook('onRoute', (route) => {
+      config.dyn.addRoute(route.url);
+    });
+
+    fastify.register(indexRoute);
+    fastify.register(getRoute);
+    fastify.register(configurationRoute);
+
+    config.env.third_party.searx_url && fastify.register(redirectRoute);
+    config.env.proxy.enabled && fastify.register(proxyRoute);
+
+    fastify.register(parseRoute);
+    fastify.register(rawHtml);
+
+    fastify.listen({ host: config.env.host, port: config.env.port }, (err) => {
+      err && console.log(err);
+    });
+  }
+}
+
+const app = new App();
+app.init();

packages/server/src/config/dynConfig.ts

@@ -0,0 +1,10 @@
+class DynConfig {
+  public routes: Set<string> = new Set();
+  constructor() {}
+  addRoute(route: string) {
+    this.routes.add(route);
+  }
+}
+
+const dyn_config = new DynConfig();
+export default dyn_config;

packages/server/src/config/envConfig.ts

@@ -0,0 +1,51 @@
+import { config as dconfig } from 'dotenv';
+
+class EnvConfig {
+  public readonly host: string;
+  public readonly port: number;
+  public readonly timeout: number;
+  public readonly reverse_proxy: boolean;
+  public readonly proxy: ProxyConfig;
+  public readonly swagger: boolean;
+  public readonly third_party: ThirdPartyConfig;
+
+  constructor() {
+    dconfig();
+
+    this.host = process.env.HOST || '0.0.0.0';
+    this.port = Number(process.env.PORT) || 8080;
+
+    this.timeout = Number(process.env.TIMEOUT) || 0;
+
+    this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false);
+
+    this.proxy = {
+      enabled: this.parseBool(process.env.PROXY_RES, true),
+      img_compress: this.parseBool(process.env.IMG_COMPRESS, true),
+    };
+
+    this.swagger = this.parseBool(process.env.SWAGGER, false);
+
+    this.third_party = {
+      searx_url: process.env.SEARX_URL,
+      webder_url: process.env.WEBDER_URL,
+    };
+  }
+
+  parseBool(value: string | undefined, def: boolean): boolean {
+    if (!value) return def;
+    return value === 'true' || value === '1';
+  }
+}
+const env_config = new EnvConfig();
+export default env_config;
+
+interface ProxyConfig {
+  enabled: boolean;
+  img_compress: boolean;
+}
+
+interface ThirdPartyConfig {
+  searx_url?: string;
+  webder_url?: string;
+}

packages/server/src/config/index.ts

@@ -0,0 +1,13 @@
+import dyn_config from './dynConfig';
+import env_config from './envConfig';
+import package_config from './packageConfig';
+import plugin_config from './pluginConfig';
+
+const config = {
+  dyn: dyn_config,
+  env: env_config,
+  plugin: plugin_config,
+  package: package_config,
+};
+
+export default config;

packages/server/src/config/packageConfig.ts

@@ -0,0 +1,3 @@
+import * as package_config from '../../package.json';
+
+export default package_config;

packages/server/src/config/pluginConfig.ts

@@ -0,0 +1,12 @@
+import { IAppConfig } from '../types/appConfig';
+import { engineList } from '@txtdot/plugins';
+
+/**
+ * Configuration of plugins
+ * Here you can add your own plugins
+ */
+const plugin_config: IAppConfig = {
+  engines: [...engineList],
+};
+
+export default plugin_config;

packages/server/src/distributor.ts

@@ -0,0 +1,86 @@
+import axios, { oaxios } from './types/axios';
+import micromatch from 'micromatch';
+import DOMPurify from 'dompurify';
+import { Readable } from 'stream';
+import { NotHtmlMimetypeError } from './errors/main';
+import { decodeStream, parseEncodingName } from './utils/http';
+import replaceHref from './utils/replace-href';
+import { parseHTML } from 'linkedom';
+
+import { Engine } from '@txtdot/sdk';
+import { HandlerInput, IHandlerOutput } from '@txtdot/sdk/dist/types/handler';
+import config from './config';
+
+interface IEngineId {
+  [key: string]: number;
+}
+
+export class Distributor {
+  engines_id: IEngineId = {};
+  fallback: Engine[] = [];
+  list: string[] = [];
+  constructor() {}
+
+  engine(engine: Engine) {
+    this.engines_id[engine.name] = this.list.length;
+    this.fallback.push(engine);
+    this.list.push(engine.name);
+  }
+
+  async handlePage(
+    remoteUrl: string, // remote URL
+    requestUrl: URL, // proxy URL
+    engineName?: string,
+    redirectPath: string = 'get'
+  ): Promise<IHandlerOutput> {
+    const urlObj = new URL(remoteUrl);
+
+    const webder_url = config.env.third_party.webder_url;
+
+    const response = webder_url
+      ? await oaxios.get(
+          `${webder_url}/render?url=${encodeURIComponent(remoteUrl)}`
+        )
+      : await axios.get(remoteUrl);
+
+    const data: Readable = response.data;
+    const mime: string | undefined =
+      response.headers['content-type']?.toString();
+
+    if (mime && mime.indexOf('text/html') === -1) {
+      throw new NotHtmlMimetypeError();
+    }
+
+    const engine = this.getFallbackEngine(urlObj.hostname, engineName);
+    const output = await engine.handle(
+      new HandlerInput(
+        await decodeStream(data, parseEncodingName(mime)),
+        remoteUrl
+      )
+    );
+
+    // post-process
+    // TODO: generate dom in handler and not parse here twice
+    const dom = parseHTML(output.content);
+    replaceHref(dom, requestUrl, new URL(remoteUrl), engineName, redirectPath);
+
+    const purify = DOMPurify(dom.window);
+    output.content = purify.sanitize(dom.document.toString());
+
+    return output;
+  }
+
+  getFallbackEngine(host: string, specified?: string): Engine {
+    if (specified) {
+      return this.fallback[this.engines_id[specified]];
+    }
+
+    for (const engine of this.fallback) {
+      if (micromatch.isMatch(host, engine.domains)) {
+        return engine;
+      }
+    }
+
+    return this.fallback[0];
+  }
+}

packages/server/src/errors/api.ts

@@ -0,0 +1,34 @@
+export interface IApiError {
+  code: number;
+  name: string;
+  message: string;
+}
+
+export const errorSchema = {
+  type: 'object',
+  properties: {
+    code: {
+      type: 'number',
+      description: 'HTTP error code',
+    },
+    name: {
+      type: 'string',
+      description: 'Exception class name',
+    },
+    message: {
+      type: 'string',
+      description: 'Exception message',
+    },
+  },
+};
+
+export const errorResponseSchema = {
+  type: 'object',
+  properties: {
+    data: {
+      type: 'object',
+      nullable: true,
+    },
+    error: errorSchema,
+  },
+};

packages/server/src/errors/handler.ts

@@ -0,0 +1,70 @@
+import { FastifyReply, FastifyRequest } from 'fastify';
+import { NotHtmlMimetypeError } from './main';
+import { getFastifyError } from './validation';
+
+import { TxtDotError } from '@txtdot/sdk/dist/types/errors';
+
+import { IGetSchema } from '../types/requests/browser';
+import config from '../config';
+
+export default function errorHandler(
+  error: Error,
+  req: FastifyRequest,
+  reply: FastifyReply
+) {
+  if (req.originalUrl.startsWith('/api/')) {
+    return apiErrorHandler(error, reply);
+  }
+
+  const url = (req as FastifyRequest<IGetSchema>).query.url;
+  return htmlErrorHandler(error, reply, url);
+}
+
+function apiErrorHandler(error: Error, reply: FastifyReply) {
+  function generateResponse(code: number) {
+    return reply.code(code).send({
+      data: null,
+      error: {
+        code: code,
+        name: error.name,
+        message: error.message,
+      },
+    });
+  }
+
+  if (getFastifyError(error)?.statusCode === 400) {
+    return generateResponse(400);
+  }
+
+  if (error instanceof TxtDotError) {
+    return generateResponse(error.code);
+  }
+
+  return generateResponse(500);
+}
+
+function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) {
+  if (getFastifyError(error)?.statusCode === 400) {
+    return reply.code(400).view('/templates/error.ejs', {
+      url,
+      code: 400,
+      description: `Invalid parameter specified: ${error.message}`,
+    });
+  }
+
+  if (error instanceof TxtDotError) {
+    return reply.code(error.code).view('/templates/error.ejs', {
+      url,
+      code: error.code,
+      description: error.description,
+      proxyBtn:
+        error instanceof NotHtmlMimetypeError && config.env.proxy.enabled,
+    });
+  }
+
+  return reply.code(500).view('/templates/error.ejs', {
+    url,
+    code: 500,
+    description: `${error.name}: ${error.message}`,
+  });
+}

packages/server/src/errors/main.ts

@@ -0,0 +1,31 @@
+import config from '../config';
+import { TxtDotError } from '@txtdot/sdk/dist/types/errors';
+
+export class LocalResourceError extends TxtDotError {
+  constructor() {
+    super(403, 'LocalResourceError', 'Proxying local resources is forbidden.');
+  }
+}
+
+export class UnsupportedMimetypeError extends TxtDotError {
+  constructor(expected: string, got?: string) {
+    super(
+      415,
+      'UnsupportedMimetypeError',
+      `Unsupported mimetype, expected ${expected}, got ${got}`
+    );
+  }
+}
+
+export class NotHtmlMimetypeError extends TxtDotError {
+  constructor() {
+    super(
+      421,
+      'NotHtmlMimetypeError',
+      'Received non-HTML content, ' +
+        (config.env.proxy.enabled
+          ? 'use proxy instead of parser.'
+          : 'proxying is disabled by the instance admin.')
+    );
+  }
+}

packages/server/src/errors/validation.ts

@@ -0,0 +1,8 @@
+export interface IFastifyValidationError {
+  statusCode?: number;
+  code?: string;
+}
+
+export function getFastifyError(error: Error) {
+  return error as unknown as IFastifyValidationError;
+}

packages/server/src/plugin_manager.ts

@@ -0,0 +1,11 @@
+import { Distributor } from './distributor';
+import plugin_config from './config/pluginConfig';
+
+const distributor = new Distributor();
+
+for (const engine of plugin_config.engines) {
+  distributor.engine(engine);
+}
+
+export const engineList = distributor.list;
+export { distributor };

packages/server/src/routes/api/parse.ts

@@ -0,0 +1,31 @@
+import { FastifyInstance } from 'fastify';
+
+import {
+  EngineRequest,
+  IParseSchema,
+  parseSchema,
+} from '../../types/requests/api';
+
+import { distributor } from '../../plugin_manager';
+import { generateRequestUrl } from '../../utils/generate';
+
+export default async function parseRoute(fastify: FastifyInstance) {
+  fastify.get<IParseSchema>(
+    '/api/parse',
+    { schema: parseSchema },
+    async (request: EngineRequest) => {
+      return {
+        data: await distributor.handlePage(
+          request.query.url,
+          generateRequestUrl(
+            request.protocol,
+            request.hostname,
+            request.originalUrl
+          ),
+          request.query.engine
+        ),
+        error: null,
+      };
+    }
+  );
+}

packages/server/src/routes/api/raw-html.ts

@@ -0,0 +1,28 @@
+import { FastifyInstance } from 'fastify';
+
+import { IParseSchema, rawHtmlSchema } from '../../types/requests/api';
+
+import { distributor } from '../../plugin_manager';
+import { generateRequestUrl } from '../../utils/generate';
+
+export default async function rawHtml(fastify: FastifyInstance) {
+  fastify.get<IParseSchema>(
+    '/api/raw-html',
+    { schema: rawHtmlSchema },
+    async (request, reply) => {
+      reply.type('text/html; charset=utf-8');
+      return (
+        await distributor.handlePage(
+          request.query.url,
+          generateRequestUrl(
+            request.protocol,
+            request.hostname,
+            request.originalUrl
+          ),
+          request.query.engine,
+          'api/raw-html'
+        )
+      ).content;
+    }
+  );
+}

packages/server/src/routes/browser/configuration.ts

@@ -0,0 +1,15 @@
+import { FastifyInstance } from 'fastify';
+
+import { distributor } from '../../plugin_manager';
+import { indexSchema } from '../../types/requests/browser';
+
+import config from '../../config';
+
+export default async function configurationRoute(fastify: FastifyInstance) {
+  fastify.get('/configuration', { schema: indexSchema }, async (_, reply) => {
+    return reply.view('/templates/configuration.ejs', {
+      engines: distributor.fallback,
+      config,
+    });
+  });
+}

packages/server/src/routes/browser/get.ts

@@ -0,0 +1,39 @@
+import { FastifyInstance } from 'fastify';
+
+import { GetSchema, IGetSchema } from '../../types/requests/browser';
+import { distributor } from '../../plugin_manager';
+import { generateRequestUrl } from '../../utils/generate';
+import config from '../../config';
+
+export default async function getRoute(fastify: FastifyInstance) {
+  fastify.get<IGetSchema>(
+    '/get',
+    { schema: GetSchema },
+    async (request, reply) => {
+      const remoteUrl = request.query.url;
+      const engine = request.query.engine;
+
+      const parsed = await distributor.handlePage(
+        remoteUrl,
+        generateRequestUrl(
+          request.protocol,
+          request.hostname,
+          request.originalUrl
+        ),
+        engine
+      );
+
+      if (request.query.format === 'text') {
+        reply.type('text/plain; charset=utf-8');
+        return parsed.textContent;
+      } else {
+        reply.type('text/html; charset=utf-8');
+        return reply.view('/templates/get.ejs', {
+          parsed,
+          remoteUrl,
+          config,
+        });
+      }
+    }
+  );
+}

packages/server/src/routes/browser/index.ts

@@ -0,0 +1,14 @@
+import { FastifyInstance } from 'fastify';
+
+import { engineList } from '../../plugin_manager';
+import { indexSchema } from '../../types/requests/browser';
+import config from '../../config';
+
+export default async function indexRoute(fastify: FastifyInstance) {
+  fastify.get('/', { schema: indexSchema }, async (_, reply) => {
+    return reply.view('/templates/index.ejs', {
+      engineList,
+      config,
+    });
+  });
+}

packages/server/src/routes/browser/proxy.ts

@@ -0,0 +1,68 @@
+import { FastifyInstance } from 'fastify';
+import { IProxySchema, ProxySchema } from '../../types/requests/browser';
+import axios from '../../types/axios';
+import sharp from 'sharp';
+import { UnsupportedMimetypeError } from '../../errors/main';
+import config from '../../config';
+
+export default async function proxyRoute(fastify: FastifyInstance) {
+  fastify.get<IProxySchema>(
+    '/proxy',
+    { schema: ProxySchema },
+    async (request, reply) => {
+      const response = await axios.get(request.query.url);
+      const mime: string | undefined =
+        response.headers['content-type']?.toString();
+      const clen: string | undefined =
+        response.headers['content-length']?.toString();
+      mime && reply.header('Content-Type', mime);
+      clen && reply.header('Content-Length', Number(clen));
+      return reply.send(response.data);
+    }
+  );
+
+  if (config.env.proxy.img_compress)
+    fastify.get<IProxySchema>(
+      '/proxy/img',
+      { schema: ProxySchema },
+      async (request, reply) => {
+        const response = await axios.get(request.query.url, {
+          responseType: 'arraybuffer',
+        });
+
+        const mime: string | undefined =
+          response.headers['content-type']?.toString();
+
+        if (!(mime && mime.startsWith('image/'))) {
+          throw new UnsupportedMimetypeError('image/*', mime);
+        }
+
+        const clen: number | undefined = parseInt(
+          response.headers['content-length']?.toString() || '0'
+        );
+
+        if (mime.startsWith('image/svg')) {
+          reply.header('Content-Type', mime);
+          reply.header('Content-Length', clen);
+          return reply.send(response.data);
+        }
+
+        const buffer = await sharp(response.data)
+          // .grayscale(true)
+          .toFormat('webp', {
+            quality: 25,
+            progressive: true,
+            optimizeScans: true,
+          })
+          .toBuffer();
+
+        reply.header('Content-Type', 'image/webp');
+        reply.header('Content-Length', buffer.length);
+
+        reply.header('x-original-size', clen);
+        reply.header('x-bytes-saved', clen - buffer.length);
+
+        return reply.send(buffer);
+      }
+    );
+}

packages/server/src/routes/browser/redirect.ts

@@ -0,0 +1,20 @@
+import { FastifyInstance } from 'fastify';
+
+import { redirectSchema, IRedirectSchema } from '../../types/requests/browser';
+
+export default async function redirectRoute(fastify: FastifyInstance) {
+  fastify.get<IRedirectSchema>(
+    '/redirect',
+    { schema: redirectSchema },
+    async (request, reply) => {
+      const params = new URLSearchParams(request.query);
+      params.delete('url');
+
+      reply.redirect(
+        `/get?url=${encodeURIComponent(
+          request.query.url + '?' + params.toString()
+        )}`
+      );
+    }
+  );
+}

packages/server/src/types/appConfig.ts

@@ -0,0 +1,5 @@
+import { Engine } from '@txtdot/sdk';
+
+export interface IAppConfig {
+  engines: Engine[];
+}

packages/server/src/types/axios.ts

@@ -0,0 +1,41 @@
+import origAxios, { CreateAxiosDefaults } from 'axios';
+import { isLocalResource, isLocalResourceURL } from '../utils/islocal';
+import { LocalResourceError } from '../errors/main';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const config: CreateAxiosDefaults<any> = {
+  headers: {
+    'User-Agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0',
+  },
+  responseType: 'stream',
+};
+
+const axios = origAxios.create(config);
+
+axios.interceptors.response.use(
+  (response) => {
+    if (isLocalResource(response.request.socket.remoteAddress)) {
+      throw new LocalResourceError();
+    }
+
+    return response;
+  },
+  async (error) => {
+    if (await isLocalResourceURL(new URL(error.config?.url))) {
+      throw new LocalResourceError();
+    }
+
+    throw error;
+  }
+);
+
+/**
+ * Modified axios for blocking local resources
+ */
+export default axios;
+
+/**
+ *  Original axios
+ */
+export const oaxios = origAxios.create(config);

packages/server/src/types/requests/api.ts

@@ -0,0 +1,62 @@
+import { FastifySchema, FastifyRequest } from 'fastify';
+import { IApiError, errorResponseSchema } from '../../errors/api';
+import { engineList } from '../../plugin_manager';
+import { FromSchema } from 'json-schema-to-ts';
+import { handlerSchema } from '@txtdot/sdk/dist/types/handler';
+
+export interface IApiResponse<T> {
+  data?: T;
+  error?: IApiError;
+}
+
+export const parseQuerySchema = {
+  type: 'object',
+  required: ['url'],
+  properties: {
+    url: {
+      type: 'string',
+      description: 'URL',
+    },
+    engine: {
+      type: 'string',
+      enum: [...engineList, ''],
+    },
+  },
+} as const;
+
+export const parseSchema: FastifySchema = {
+  description: 'Parse the page and get all data from the engine',
+  querystring: parseQuerySchema,
+  response: {
+    '2xx': {
+      type: 'object',
+      properties: {
+        data: handlerSchema,
+        error: {
+          type: 'object',
+          nullable: true,
+        },
+      },
+    },
+    '4xx': errorResponseSchema,
+    '5xx': errorResponseSchema,
+  },
+  produces: ['text/json'],
+};
+
+export interface IParseSchema {
+  Querystring: FromSchema<typeof parseQuerySchema>;
+}
+
+export const rawHtmlSchema: FastifySchema = {
+  description: 'Parse the page and get raw HTML from the engine',
+  querystring: parseQuerySchema,
+  produces: ['text/html'],
+};
+
+export type EngineRequest = FastifyRequest<{
+  Querystring: {
+    url: string;
+    engine?: string;
+  };
+}>;

packages/server/src/types/requests/browser.ts

@@ -0,0 +1,90 @@
+import { FastifySchema } from 'fastify';
+import { engineList } from '../../plugin_manager';
+import { FromSchema } from 'json-schema-to-ts';
+
+export interface IGetSchema {
+  Querystring: IGetQuerySchema;
+}
+
+export interface IProxySchema {
+  Querystring: IProxyQuerySchema;
+}
+
+export interface IRedirectSchema {
+  Querystring: IRedirectQuerySchema;
+}
+
+export const redirectQuerySchema = {
+  type: 'object',
+  required: ['url'],
+  properties: {
+    url: {
+      type: 'string',
+      description: 'URL to redirect without querystring',
+    },
+  },
+  patternProperties: {
+    '^(?!url).*$': { type: 'string' },
+  },
+} as const;
+export type IRedirectQuerySchema = {
+  url: string;
+  [key: string]: string;
+};
+
+export const getQuerySchema = {
+  type: 'object',
+  required: ['url'],
+  properties: {
+    url: {
+      type: 'string',
+      description: 'URL',
+    },
+    format: {
+      type: 'string',
+      enum: ['text', 'html', ''],
+      default: 'html',
+    },
+    engine: {
+      type: 'string',
+      enum: [...engineList, ''],
+    },
+  },
+} as const;
+export type IGetQuerySchema = FromSchema<typeof getQuerySchema>;
+
+export const proxyQuerySchema = {
+  type: 'object',
+  required: ['url'],
+  properties: {
+    url: {
+      type: 'string',
+      description: 'URL',
+    },
+  },
+} as const;
+export type IProxyQuerySchema = FromSchema<typeof proxyQuerySchema>;
+
+export const indexSchema = {
+  hide: true,
+  produces: ['text/html'],
+};
+
+export const redirectSchema: FastifySchema = {
+  description: 'Universal redirection page',
+  hide: true,
+  querystring: redirectQuerySchema,
+};
+
+export const GetSchema: FastifySchema = {
+  description: 'Get page',
+  hide: true,
+  querystring: getQuerySchema,
+  produces: ['text/html', 'text/plain'],
+};
+
+export const ProxySchema: FastifySchema = {
+  description: 'Proxy resource',
+  hide: true,
+  querystring: proxyQuerySchema,
+};

packages/server/src/utils/generate.ts

@@ -0,0 +1,43 @@
+export function generateRequestUrl(
+  protocol: string,
+  host: string,
+  originalUrl: string
+): URL {
+  return new URL(`${protocol}://${host}${originalUrl}`);
+}
+
+export function generateParserUrl(
+  requestUrl: URL,
+  remoteUrl: URL,
+  href: string,
+  engine?: string,
+  redirect_url: string = 'get'
+): string {
+  const realURL = getRealURL(href, remoteUrl);
+
+  const hash = realURL.hash; // save #hash
+  realURL.hash = ''; // remove
+
+  const urlParam = `?url=${encodeURIComponent(realURL.toString())}`;
+  const engineParam = engine ? `&engine=${engine}` : '';
+
+  return `${requestUrl.origin}/${redirect_url}${urlParam}${engineParam}${hash}`;
+}
+
+export function generateProxyUrl(
+  requestUrl: URL,
+  remoteUrl: URL,
+  href: string,
+  subProxy?: string
+): string {
+  const realHref = getRealURL(href, remoteUrl);
+
+  const urlParam = `?url=${encodeURIComponent(realHref.href)}`;
+  return `${requestUrl.origin}/proxy${subProxy ? `/${subProxy}` : ''}${urlParam}`;
+}
+
+function getRealURL(href: string, remoteUrl: URL) {
+  return href.startsWith('http')
+    ? new URL(href)
+    : new URL(href, remoteUrl.href);
+}

packages/server/src/utils/http.ts

@@ -0,0 +1,26 @@
+import { Readable } from 'stream';
+import iconv from 'iconv-lite';
+
+export async function decodeStream(
+  data: Readable,
+  charset: string = 'utf-8'
+): Promise<string> {
+  const strm = data.pipe(iconv.decodeStream(charset)) as IconvStream;
+  return await new Promise((resolve) => {
+    strm.collect((_err: Error, body: string) => {
+      resolve(body);
+    });
+  });
+}
+
+export function parseEncodingName(ctype?: string): string {
+  const match = ctype?.match(/charset=([A-Za-z0-9-]+)$/);
+  if (!match) {
+    return 'utf-8';
+  }
+  return match[1];
+}
+
+interface IconvStream extends NodeJS.ReadWriteStream {
+  collect: (cb: (err: Error, body: string) => void) => void;
+}

packages/server/src/utils/islocal.ts

@@ -0,0 +1,48 @@
+import ipRangeCheck from 'ip-range-check';
+import dns from 'dns';
+
+const subnets = [
+  '0.0.0.0/8',
+  '127.0.0.0/8',
+  '10.0.0.0/8',
+  '100.64.0.0/10',
+  '169.254.0.0/16',
+  '172.16.0.0/12',
+  '192.0.0.0/24',
+  '192.0.2.0/24',
+  '192.88.99.0/24',
+  '192.168.0.0/16',
+  '198.18.0.0/15',
+  '198.51.100.0/24',
+  '203.0.113.0/24',
+  '224.0.0.0/4',
+  '233.252.0.0/24',
+  '240.0.0.0/4',
+  '255.255.255.255/32',
+  '::/128',
+  '::1/128',
+  '::ffff:0:0/96',
+  '::ffff:0:0:0/96',
+  '64:ff9b::/96',
+  '64:ff9b:1::/48',
+  '100::/64',
+  '2001:0000::/32',
+  '2001:20::/28',
+  '2001:db8::/32',
+  '2002::/16',
+  'fc00::/7',
+  'fe80::/64',
+  'ff00::/8',
+];
+
+export function isLocalResource(addr: string): boolean {
+  return ipRangeCheck(addr, subnets);
+}
+
+export async function isLocalResourceURL(url: URL): Promise<boolean> {
+  // Resolve domain name
+  const addr = (await dns.promises.lookup(url.hostname)).address;
+
+  // Check if IP is in local network
+  return ipRangeCheck(addr, subnets);
+}

packages/server/src/utils/replace-href.ts

@@ -0,0 +1,77 @@
+import config from '../config';
+import { generateParserUrl, generateProxyUrl } from './generate';
+
+export default function replaceHref(
+  dom: Window,
+  requestUrl: URL,
+  remoteUrl: URL,
+  engine?: string,
+  redirectPath: string = 'get'
+) {
+  const doc: Document = dom.window.document;
+  const parserUrl = (href: string) =>
+    generateParserUrl(requestUrl, remoteUrl, href, engine, redirectPath);
+
+  const proxyUrl = (href: string) =>
+    generateProxyUrl(requestUrl, remoteUrl, href);
+
+  const imgProxyUrl = (href: string) =>
+    generateProxyUrl(requestUrl, remoteUrl, href, 'img');
+
+  modifyLinks(doc.querySelectorAll('a[href]'), 'href', parserUrl);
+  modifyLinks(doc.querySelectorAll('frame,iframe'), 'src', parserUrl);
+
+  if (config.env.proxy.enabled) {
+    modifyLinks(
+      doc.querySelectorAll('video,audio,embed,track,source'),
+      'src',
+      proxyUrl
+    );
+
+    modifyLinks(
+      doc.querySelectorAll('img,image'),
+      'src',
+      config.env.proxy.img_compress ? imgProxyUrl : proxyUrl
+    );
+
+    modifyLinks(doc.getElementsByTagName('object'), 'data', proxyUrl);
+    const sources = doc.querySelectorAll('source,img');
+    for (const source of sources) {
+      // split srcset by comma
+      // @ts-expect-error because I don't know what to do about it.
+      if (!source.srcset) continue;
+      // @ts-expect-error because I don't know what to do about it.
+      source.srcset = source.srcset
+        .split(',')
+        .map((src: string) => {
+          // split src by space
+          const parts = src.trim().split(' ');
+          try {
+            // first part is URL
+            // (srcset="http 200w 1x,...")
+            parts[0] = proxyUrl(parts[0]);
+          } catch (_err) {
+            /* empty */
+          }
+          // join by space after splitting
+          return parts.join(' ');
+        })
+        .join(','); // join by comma
+    }
+  }
+}
+
+function modifyLinks(
+  nodeList: NodeListOf<Element> | HTMLCollectionOf<Element>,
+  property: string,
+  generateLink: (value: string) => string
+) {
+  for (const node of nodeList) {
+    try {
+      // @ts-expect-error because I don't know what to do about it.
+      node[property] = generateLink(node[property]);
+    } catch (_err) {
+      /* empty */
+    }
+  }
+}