Feature: allow disable ipv6 in proxy, refactor cacheFetch to use proxy (#5011)

docs/troubleshooting/index.md

@@ -70,7 +70,9 @@ If, after correctly adding and mapping your custom icons via the [Icons](../conf
 
 ## Disabling IPv6
 
-If you are having issues with certain widgets that are unable to reach public APIs (e.g. weather), you may need to disable IPv6 on your host machine. This can be done by adding the following to your `docker-compose.yml` file (or for docker run, the equivalent flag):
+If you are having issues with certain widgets that are unable to reach public APIs (e.g. weather), in certain setups you may need to disable IPv6. You can set the environment variable `HOMEPAGE_PROXY_DISABLE_IPV6` to `true` to disable IPv6 for the homepage proxy.
+
+Alternatively, you can use the `sysctls` option in your docker-compose file to disable IPv6 for the homepage container completely:
 
 ```yaml
 services:
@@ -79,12 +81,3 @@ services:
     sysctls:
       - net.ipv6.conf.all.disable_ipv6=1
 ```
-
-or disable IPv6 for the docker network:
-
-```yaml
-networks:
-  some_network:
-    driver: bridge
-    enable_ipv6: false
-```

src/pages/api/releases.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import createLogger from "utils/logger";
 
 const logger = createLogger("releases");
@@ -6,7 +6,7 @@ const logger = createLogger("releases");
 export default async function handler(req, res) {
   const releasesURL = "https://api.github.com/repos/gethomepage/homepage/releases";
   try {
-    return res.send(await cachedFetch(releasesURL, 5));
+    return res.send(await cachedRequest(releasesURL, 5));
   } catch (e) {
     logger.error(`Error checking GitHub releases: ${e}`);
     return res.send([]);

src/pages/api/search/searchSuggestion.js

@@ -1,7 +1,7 @@
 import { searchProviders } from "components/widgets/search/search";
 
 import { getSettings } from "utils/config/config";
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { widgetsFromConfig } from "utils/config/widget-helpers";
 
 export default async function handler(req, res) {
@@ -29,5 +29,5 @@ export default async function handler(req, res) {
     return res.json([query, []]); // Responde with the same array format but with no suggestions.
   }
 
-  return res.send(await cachedFetch(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0"));
+  return res.send(await cachedRequest(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0"));
 }

src/pages/api/widgets/openmeteo.js

@@ -1,9 +1,9 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 
 export default async function handler(req, res) {
   const { latitude, longitude, units, cache, timezone } = req.query;
   const degrees = units === "metric" ? "celsius" : "fahrenheit";
   const timezeone = timezone ?? "auto";
   const apiUrl = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&daily=sunrise,sunset&current_weather=true&temperature_unit=${degrees}&timezone=${timezeone}`;
-  return res.send(await cachedFetch(apiUrl, cache));
+  return res.send(await cachedRequest(apiUrl, cache));
 }

src/pages/api/widgets/openweathermap.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { getSettings } from "utils/config/config";
 import { getPrivateWidgetOptions } from "utils/config/widget-helpers";
 
@@ -26,5 +26,5 @@ export default async function handler(req, res) {
 
   const apiUrl = `https://api.openweathermap.org/data/2.5/weather?lat=${latitude}&lon=${longitude}&appid=${apiKey}&units=${units}&lang=${lang}`;
 
-  return res.send(await cachedFetch(apiUrl, cache));
+  return res.send(await cachedRequest(apiUrl, cache));
 }

src/pages/api/widgets/stocks.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { getSettings } from "utils/config/config";
 import createLogger from "utils/logger";
 
@@ -60,7 +60,7 @@ export default async function handler(req, res) {
         const apiUrl = `https://finnhub.io/api/v1/quote?symbol=${ticker}&token=${apiKey}`;
         // Finnhub free accounts allow up to 60 calls/minute
         // https://finnhub.io/pricing
-        const { c, dp } = await cachedFetch(apiUrl, cache || 1);
+        const { c, dp } = await cachedRequest(apiUrl, cache || 1);
         logger.debug("Finnhub API response for %s: %o", ticker, { c, dp });
 
         // API sometimes returns 200, but values returned are `null`

src/pages/api/widgets/weather.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { getSettings } from "utils/config/config";
 import { getPrivateWidgetOptions } from "utils/config/widget-helpers";
 
@@ -26,5 +26,5 @@ export default async function handler(req, res) {
 
   const apiUrl = `http://api.weatherapi.com/v1/current.json?q=${latitude},${longitude}&key=${apiKey}&lang=${lang}`;
 
-  return res.send(await cachedFetch(apiUrl, cache));
+  return res.send(await cachedRequest(apiUrl, cache));
 }

src/utils/config/widget-helpers.js

@@ -56,7 +56,8 @@ export async function cleanWidgetGroups(widgets) {
 export async function getPrivateWidgetOptions(type, widgetIndex) {
   const widgets = await widgetsFromConfig();
 
-  const privateOptions = widgets.map((widget) => {
+  const privateOptions =
+    widgets.map((widget) => {
       const { index, url, username, password, key, apiKey } = widget.options;
 
       return {
@@ -70,7 +71,7 @@ export async function getPrivateWidgetOptions(type, widgetIndex) {
           apiKey,
         },
       };
-  });
+    }) || {};
 
   return type !== undefined && widgetIndex !== undefined
     ? privateOptions.find((o) => o.type === type && o.options.index === parseInt(widgetIndex, 10))?.options

src/utils/proxy/cached-fetch.js

@@ -1,25 +0,0 @@
-import cache from "memory-cache";
-
-const defaultDuration = 5;
-
-export default async function cachedFetch(url, duration, ua) {
-  const cached = cache.get(url);
-
-  // eslint-disable-next-line no-param-reassign
-  duration = duration || defaultDuration;
-
-  if (cached) {
-    return cached;
-  }
-
-  // wrapping text in JSON.parse to handle utf-8 issues
-  const options = {};
-  if (ua) {
-    options.headers = {
-      "User-Agent": ua,
-    };
-  }
-  const data = await fetch(url, options).then((res) => res.json());
-  cache.put(url, data, duration * 1000 * 60);
-  return data;
-}

src/utils/proxy/http.js

@@ -3,6 +3,7 @@
 import { createUnzip, constants as zlibConstants } from "node:zlib";
 
 import { http, https } from "follow-redirects";
+import cache from "memory-cache";
 
 import { addCookieToJar, setCookieHeader } from "./cookie-jar";
 import { sanitizeErrorURL } from "./api-helpers";
@@ -81,20 +82,46 @@ export function httpRequest(url, params) {
   return handleRequest(http, url, params);
 }
 
+export async function cachedRequest(url, duration = 5, ua = "homepage") {
+  const cached = cache.get(url);
+
+  if (cached) {
+    return cached;
+  }
+
+  const options = {
+    headers: {
+      "User-Agent": ua,
+      Accept: "application/json",
+    },
+  };
+  let [, , data] = await httpProxy(url, options);
+  if (Buffer.isBuffer(data)) {
+    try {
+      data = JSON.parse(Buffer.from(data).toString());
+    } catch (e) {
+      logger.debug("Error parsing cachedRequest data for %s: %s %s", url, Buffer.from(data).toString(), e);
+      data = Buffer.from(data).toString();
+    }
+  }
+  cache.put(url, data, duration * 1000 * 60);
+  return data;
+}
+
 export async function httpProxy(url, params = {}) {
   const constructedUrl = new URL(url);
+  const disableIpv6 = process.env.HOMEPAGE_PROXY_DISABLE_IPV6 === "true";
+  const agentOptions = disableIpv6 ? { family: 4, autoSelectFamily: false } : {};
 
   let request = null;
   if (constructedUrl.protocol === "https:") {
     request = httpsRequest(constructedUrl, {
-      agent: new https.Agent({
-        rejectUnauthorized: false,
-      }),
+      agent: new https.Agent({ ...agentOptions, rejectUnauthorized: false }),
       ...params,
     });
   } else {
     request = httpRequest(constructedUrl, {
-      agent: new http.Agent(),
+      agent: new http.Agent(agentOptions),
       ...params,
     });
   }