From b4dc53c7c0e07ba4eaa59a4b3dd698ae11fa1f32 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Sun, 16 Mar 2025 20:09:34 -0700 Subject: [PATCH] Feature: allow disable ipv6 in proxy, refactor cacheFetch to use proxy (#5011) --- docs/troubleshooting/index.md | 13 ++------- src/pages/api/releases.js | 4 +-- src/pages/api/search/searchSuggestion.js | 4 +-- src/pages/api/widgets/openmeteo.js | 4 +-- src/pages/api/widgets/openweathermap.js | 4 +-- src/pages/api/widgets/stocks.js | 4 +-- src/pages/api/widgets/weather.js | 4 +-- src/utils/config/widget-helpers.js | 29 ++++++++++---------- src/utils/proxy/cached-fetch.js | 25 ----------------- src/utils/proxy/http.js | 35 +++++++++++++++++++++--- 10 files changed, 61 insertions(+), 65 deletions(-) delete mode 100644 src/utils/proxy/cached-fetch.js diff --git a/docs/troubleshooting/index.md b/docs/troubleshooting/index.md index 7f73f2dd..81550439 100644 --- a/docs/troubleshooting/index.md +++ b/docs/troubleshooting/index.md @@ -70,7 +70,9 @@ If, after correctly adding and mapping your custom icons via the [Icons](../conf ## Disabling IPv6 -If you are having issues with certain widgets that are unable to reach public APIs (e.g. weather), you may need to disable IPv6 on your host machine. This can be done by adding the following to your `docker-compose.yml` file (or for docker run, the equivalent flag): +If you are having issues with certain widgets that are unable to reach public APIs (e.g. weather), in certain setups you may need to disable IPv6. You can set the environment variable `HOMEPAGE_PROXY_DISABLE_IPV6` to `true` to disable IPv6 for the homepage proxy. + +Alternatively, you can use the `sysctls` option in your docker-compose file to disable IPv6 for the homepage container completely: ```yaml services: @@ -79,12 +81,3 @@ services: sysctls: - net.ipv6.conf.all.disable_ipv6=1 ``` - -or disable IPv6 for the docker network: - -```yaml -networks: - some_network: - driver: bridge - enable_ipv6: false -``` diff --git a/src/pages/api/releases.js b/src/pages/api/releases.js index f15930c2..372ace9d 100644 --- a/src/pages/api/releases.js +++ b/src/pages/api/releases.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import createLogger from "utils/logger"; const logger = createLogger("releases"); @@ -6,7 +6,7 @@ const logger = createLogger("releases"); export default async function handler(req, res) { const releasesURL = "https://api.github.com/repos/gethomepage/homepage/releases"; try { - return res.send(await cachedFetch(releasesURL, 5)); + return res.send(await cachedRequest(releasesURL, 5)); } catch (e) { logger.error(`Error checking GitHub releases: ${e}`); return res.send([]); diff --git a/src/pages/api/search/searchSuggestion.js b/src/pages/api/search/searchSuggestion.js index dbe072ea..209d1f2c 100644 --- a/src/pages/api/search/searchSuggestion.js +++ b/src/pages/api/search/searchSuggestion.js @@ -1,7 +1,7 @@ import { searchProviders } from "components/widgets/search/search"; import { getSettings } from "utils/config/config"; -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { widgetsFromConfig } from "utils/config/widget-helpers"; export default async function handler(req, res) { @@ -29,5 +29,5 @@ export default async function handler(req, res) { return res.json([query, []]); // Responde with the same array format but with no suggestions. } - return res.send(await cachedFetch(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0")); + return res.send(await cachedRequest(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0")); } diff --git a/src/pages/api/widgets/openmeteo.js b/src/pages/api/widgets/openmeteo.js index e63847b4..28f2e4f0 100644 --- a/src/pages/api/widgets/openmeteo.js +++ b/src/pages/api/widgets/openmeteo.js @@ -1,9 +1,9 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; export default async function handler(req, res) { const { latitude, longitude, units, cache, timezone } = req.query; const degrees = units === "metric" ? "celsius" : "fahrenheit"; const timezeone = timezone ?? "auto"; const apiUrl = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&daily=sunrise,sunset¤t_weather=true&temperature_unit=${degrees}&timezone=${timezeone}`; - return res.send(await cachedFetch(apiUrl, cache)); + return res.send(await cachedRequest(apiUrl, cache)); } diff --git a/src/pages/api/widgets/openweathermap.js b/src/pages/api/widgets/openweathermap.js index 089ee804..3bdc7a82 100644 --- a/src/pages/api/widgets/openweathermap.js +++ b/src/pages/api/widgets/openweathermap.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { getSettings } from "utils/config/config"; import { getPrivateWidgetOptions } from "utils/config/widget-helpers"; @@ -26,5 +26,5 @@ export default async function handler(req, res) { const apiUrl = `https://api.openweathermap.org/data/2.5/weather?lat=${latitude}&lon=${longitude}&appid=${apiKey}&units=${units}&lang=${lang}`; - return res.send(await cachedFetch(apiUrl, cache)); + return res.send(await cachedRequest(apiUrl, cache)); } diff --git a/src/pages/api/widgets/stocks.js b/src/pages/api/widgets/stocks.js index 3941a773..4e9f3f55 100644 --- a/src/pages/api/widgets/stocks.js +++ b/src/pages/api/widgets/stocks.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { getSettings } from "utils/config/config"; import createLogger from "utils/logger"; @@ -60,7 +60,7 @@ export default async function handler(req, res) { const apiUrl = `https://finnhub.io/api/v1/quote?symbol=${ticker}&token=${apiKey}`; // Finnhub free accounts allow up to 60 calls/minute // https://finnhub.io/pricing - const { c, dp } = await cachedFetch(apiUrl, cache || 1); + const { c, dp } = await cachedRequest(apiUrl, cache || 1); logger.debug("Finnhub API response for %s: %o", ticker, { c, dp }); // API sometimes returns 200, but values returned are `null` diff --git a/src/pages/api/widgets/weather.js b/src/pages/api/widgets/weather.js index 9d0451ce..9e63e48d 100644 --- a/src/pages/api/widgets/weather.js +++ b/src/pages/api/widgets/weather.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { getSettings } from "utils/config/config"; import { getPrivateWidgetOptions } from "utils/config/widget-helpers"; @@ -26,5 +26,5 @@ export default async function handler(req, res) { const apiUrl = `http://api.weatherapi.com/v1/current.json?q=${latitude},${longitude}&key=${apiKey}&lang=${lang}`; - return res.send(await cachedFetch(apiUrl, cache)); + return res.send(await cachedRequest(apiUrl, cache)); } diff --git a/src/utils/config/widget-helpers.js b/src/utils/config/widget-helpers.js index 3b1355d6..93f71194 100644 --- a/src/utils/config/widget-helpers.js +++ b/src/utils/config/widget-helpers.js @@ -56,21 +56,22 @@ export async function cleanWidgetGroups(widgets) { export async function getPrivateWidgetOptions(type, widgetIndex) { const widgets = await widgetsFromConfig(); - const privateOptions = widgets.map((widget) => { - const { index, url, username, password, key, apiKey } = widget.options; + const privateOptions = + widgets.map((widget) => { + const { index, url, username, password, key, apiKey } = widget.options; - return { - type: widget.type, - options: { - index, - url, - username, - password, - key, - apiKey, - }, - }; - }); + return { + type: widget.type, + options: { + index, + url, + username, + password, + key, + apiKey, + }, + }; + }) || {}; return type !== undefined && widgetIndex !== undefined ? privateOptions.find((o) => o.type === type && o.options.index === parseInt(widgetIndex, 10))?.options diff --git a/src/utils/proxy/cached-fetch.js b/src/utils/proxy/cached-fetch.js deleted file mode 100644 index ae3c4610..00000000 --- a/src/utils/proxy/cached-fetch.js +++ /dev/null @@ -1,25 +0,0 @@ -import cache from "memory-cache"; - -const defaultDuration = 5; - -export default async function cachedFetch(url, duration, ua) { - const cached = cache.get(url); - - // eslint-disable-next-line no-param-reassign - duration = duration || defaultDuration; - - if (cached) { - return cached; - } - - // wrapping text in JSON.parse to handle utf-8 issues - const options = {}; - if (ua) { - options.headers = { - "User-Agent": ua, - }; - } - const data = await fetch(url, options).then((res) => res.json()); - cache.put(url, data, duration * 1000 * 60); - return data; -} diff --git a/src/utils/proxy/http.js b/src/utils/proxy/http.js index 3743515b..f8d2dcce 100644 --- a/src/utils/proxy/http.js +++ b/src/utils/proxy/http.js @@ -3,6 +3,7 @@ import { createUnzip, constants as zlibConstants } from "node:zlib"; import { http, https } from "follow-redirects"; +import cache from "memory-cache"; import { addCookieToJar, setCookieHeader } from "./cookie-jar"; import { sanitizeErrorURL } from "./api-helpers"; @@ -81,20 +82,46 @@ export function httpRequest(url, params) { return handleRequest(http, url, params); } +export async function cachedRequest(url, duration = 5, ua = "homepage") { + const cached = cache.get(url); + + if (cached) { + return cached; + } + + const options = { + headers: { + "User-Agent": ua, + Accept: "application/json", + }, + }; + let [, , data] = await httpProxy(url, options); + if (Buffer.isBuffer(data)) { + try { + data = JSON.parse(Buffer.from(data).toString()); + } catch (e) { + logger.debug("Error parsing cachedRequest data for %s: %s %s", url, Buffer.from(data).toString(), e); + data = Buffer.from(data).toString(); + } + } + cache.put(url, data, duration * 1000 * 60); + return data; +} + export async function httpProxy(url, params = {}) { const constructedUrl = new URL(url); + const disableIpv6 = process.env.HOMEPAGE_PROXY_DISABLE_IPV6 === "true"; + const agentOptions = disableIpv6 ? { family: 4, autoSelectFamily: false } : {}; let request = null; if (constructedUrl.protocol === "https:") { request = httpsRequest(constructedUrl, { - agent: new https.Agent({ - rejectUnauthorized: false, - }), + agent: new https.Agent({ ...agentOptions, rejectUnauthorized: false }), ...params, }); } else { request = httpRequest(constructedUrl, { - agent: new http.Agent(), + agent: new http.Agent(agentOptions), ...params, }); }