Merge pull request #26 from TxtDot/fix-google-headings

Fix google headings and proxying urls

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "txtdot",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "txtdot",
-      "version": "1.0.0",
+      "version": "1.1.1",
       "license": "MIT",
       "dependencies": {
         "@fastify/static": "^6.10.2",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "txtdot",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "private": true,
   "description": "",
   "main": "dist/app.js",

src/app.ts

@@ -27,6 +27,7 @@ class App {
   async init() {
     const fastify = Fastify({
       logger: true,
+      trustProxy: this.config.reverse_proxy_enabled,
     });
 
     fastify.register(fastifyStatic, {
@@ -58,9 +59,12 @@ class App {
 
     fastify.setErrorHandler(errorHandler);
 
-    fastify.listen({ host: this.config.host, port: this.config.port }, (err) => {
+    fastify.listen(
-      err && console.log(err);
+      { host: this.config.host, port: this.config.port },
-    });
+      (err) => {
+        err && console.log(err);
+      }
+    );
   }
 }
 

src/config/config.service.ts

@@ -3,6 +3,7 @@ import { config } from "dotenv";
 export class ConfigService {
   public readonly host: string;
   public readonly port: number;
+  public readonly reverse_proxy_enabled: boolean;
 
   constructor() {
     const parsed = config().parsed;
@@ -11,7 +12,9 @@ export class ConfigService {
       throw new Error("Invalid .env file");
     }
 
-    this.host = process.env.HOST || 'localhost';
+    this.host = process.env.HOST || "localhost";
     this.port = Number(process.env.PORT) || 8080;
+    this.reverse_proxy_enabled =
+      Boolean(process.env.REVERSE_PROXY_ENABLED) || false;
   }
 }

src/handlers/google.ts

@@ -8,17 +8,17 @@ export default async function google(
   const googleAnchors = [
     ...window.document.querySelectorAll("a[jsname=ACyKwe]"),
   ] as HTMLAnchorElement[];
-  const googleNames = [...window.document.querySelectorAll(".VuuXrf")];
 
-  const results = googleAnchors.map(
+  const results = googleAnchors
-    (a: HTMLAnchorElement, i: number): GoogleProps => {
+    .map((a: HTMLAnchorElement): GoogleProps => {
+      const parsedHref = new URL(new URL(a.href).searchParams.get("url")!);
       return {
         href: a.href!,
-        siteName: googleNames[i].textContent!,
+        siteName: parsedHref.hostname,
-        heading: a.childNodes[1].textContent!,
+        heading: a.childNodes[1]?.textContent,
       };
-    }
+    })
-  );
+    .filter((a) => a.heading);
 
   if (!googleAnchors) {
     throw new EngineParseError(
@@ -26,12 +26,6 @@ export default async function google(
     );
   }
 
-  if (!googleNames) {
-    throw new EngineParseError(
-      "Failed to find names in search result [google]"
-    );
-  }
-
   const convertToFormat = (result: GoogleProps, isHtml: boolean) => {
     return isHtml
       ? `<p><a href="${result.href}">${result.siteName} - ${result.heading}</p>`
@@ -90,5 +84,5 @@ export default async function google(
 interface GoogleProps {
   href: string;
   siteName: string;
-  heading: string;
+  heading: string | null;
 }

src/handlers/main.ts

@@ -18,6 +18,7 @@ import {
 
 export default async function handlePage(
   url: string, // remote URL
+  requestUrl: URL, // proxy URL
   engine?: string
 ): Promise<IHandlerOutput> {
   const urlObj = new URL(url);
@@ -40,7 +41,7 @@ export default async function handlePage(
   const window = new JSDOM(response.data, { url }).window;
 
   [...window.document.getElementsByTagName("a")].forEach((link) => {
-    link.href = generateProxyUrl(link.href, engine);
+    link.href = generateProxyUrl(requestUrl, link.href, engine);
   });
 
   if (engine) {

src/routes/get.ts

@@ -2,6 +2,7 @@ import { FastifyInstance } from "fastify";
 
 import { GetSchema, IGetSchema } from "../types/requests";
 import handlePage from "../handlers/main";
+import { generateRequestUrl } from "../utils/generate";
 
 export default async function getRoute(fastify: FastifyInstance) {
   fastify.get<IGetSchema>(
@@ -11,7 +12,15 @@ export default async function getRoute(fastify: FastifyInstance) {
       const remoteUrl = request.query.url;
       const engine = request.query.engine;
 
-      const parsed = await handlePage(remoteUrl, engine);
+      const parsed = await handlePage(
+        remoteUrl,
+        generateRequestUrl(
+          request.protocol,
+          request.hostname,
+          request.originalUrl
+        ),
+        engine
+      );
 
       if (request.query.format === "text") {
         reply.type("text/plain; charset=utf-8");

src/routes/parse.ts

@@ -1,13 +1,22 @@
 import { EngineRequest, IParseSchema, parseSchema } from "../types/requests";
 import { FastifyInstance } from "fastify";
 import handlePage from "../handlers/main";
+import { generateRequestUrl } from "../utils/generate";
 
 export default async function parseRoute(fastify: FastifyInstance) {
   fastify.get<IParseSchema>(
     "/parse",
     { schema: parseSchema },
     async (request: EngineRequest) => {
-      return await handlePage(request.query.url, request.query.engine);
+      return await handlePage(
+        request.query.url,
+        generateRequestUrl(
+          request.protocol,
+          request.hostname,
+          request.originalUrl
+        ),
+        request.query.engine
+      );
     }
   );
 }

src/routes/raw-html.ts

@@ -2,13 +2,23 @@ import { FastifyInstance } from "fastify";
 
 import { GetRequest, IParseSchema, rawHtmlSchema } from "../types/requests";
 import handlePage from "../handlers/main";
+import { generateRequestUrl } from "../utils/generate";
 
 export default async function rawHtml(fastify: FastifyInstance) {
   fastify.get<IParseSchema>(
     "/raw-html",
     { schema: rawHtmlSchema },
     async (request: GetRequest) => {
-      return (await handlePage(request.query.url)).content;
+      return (
+        await handlePage(
+          request.query.url,
+          generateRequestUrl(
+            request.protocol,
+            request.hostname,
+            request.originalUrl
+          )
+        )
+      ).content;
     }
   );
 }

src/utils/generate.ts

@@ -6,8 +6,12 @@ export function generateRequestUrl(
   return new URL(`${protocol}://${host}${originalUrl}`);
 }
 
-export function generateProxyUrl(href: string, engine?: string): string {
+export function generateProxyUrl(
+  requestUrl: URL,
+  href: string,
+  engine?: string
+): string {
   const urlParam = `?url=${encodeURIComponent(href)}`;
   const engineParam = engine ? `&engine=${engine}` : "";
-  return `/get${urlParam}${engineParam}`;
+  return `${requestUrl.origin}/get${urlParam}${engineParam}`;
 }