Enhancement: support for Kubernetes gateway API (#4643)

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>
Co-authored-by: lyons <gittea.sand@gmail.com>
Co-authored-by: Brett Dudo <brett@dudo.io>

docs/configs/kubernetes.md

@@ -8,6 +8,7 @@ The Kubernetes connectivity has the following requirements:
 - Kubernetes 1.19+
 - Metrics Service
 - An Ingress controller
+  - Optionally: Gateway-API
 
 The Kubernetes connection is configured in the `kubernetes.yaml` file. There are 3 modes to choose from:
 
@@ -19,6 +20,22 @@ The Kubernetes connection is configured in the `kubernetes.yaml` file. There are
 mode: default
 ```
 
+To configure Kubernetes gateway-api, ingress or ingressRoute service discovery, add one or multiple of the following settings.
+
+Example settings:
+
+```yaml
+ingress: true # enable ingress only
+```
+
+or
+
+```yaml
+ingress: true # enable ingress
+traefik: true # enable traefik ingressRoute
+gateway: true # enable gateway-api
+```
+
 ## Services
 
 Once the Kubernetes connection is configured, individual services can be configured to pull statistics. Only CPU and Memory are currently supported.
@@ -142,6 +159,10 @@ spec:
 
 If the `href` attribute is not present, Homepage will ignore the specific IngressRoute.
 
+### Gateway API HttpRoute support
+
+Homepage also features automatic service discovery for Gateway API. Service definitions are read by annotating the HttpRoute custom resource definition and are indentical to the Ingress example as defined in [Automatic Service Discovery](#automatic-service-discovery).
+
 ## Caveats
 
 Similarly to Docker service discovery, there currently is no rigid ordering to discovered services and discovered services will be displayed above those specified in the `services.yaml`.

docs/installation/k8s.md

@@ -216,6 +216,14 @@ rules:
     verbs:
       - get
       - list
+  - apiGroups:
+      - gateway.networking.k8s.io
+    resources:
+      - httproutes
+      - gateways
+    verbs:
+      - get
+      - list
   - apiGroups:
       - metrics.k8s.io
     resources:

kubernetes.md

@@ -23,6 +23,12 @@ Set the `mode` in the `kubernetes.yaml` to `cluster`.
 mode: default
 ```
 
+To enable Kubernetes gateway-api compatibility, set `route` to `gateway`.
+
+```yaml
+route: gateway
+```
+
 ## Widgets
 
 The Kubernetes widget can show a high-level overview of the cluster,

src/pages/api/kubernetes/stats/[...service].js

@@ -1,7 +1,7 @@
 import { CoreV1Api, Metrics } from "@kubernetes/client-node";
 
-import getKubeConfig from "../../../../utils/config/kubernetes";
-import { parseCpu, parseMemory } from "../../../../utils/kubernetes/kubernetes-utils";
+import { getKubeConfig } from "../../../../utils/config/kubernetes";
+import { parseCpu, parseMemory } from "../../../../utils/kubernetes/utils";
 import createLogger from "../../../../utils/logger";
 
 const logger = createLogger("kubernetesStatsService");
@@ -30,7 +30,10 @@ export default async function handler(req, res) {
     const coreApi = kc.makeApiClient(CoreV1Api);
     const metricsApi = new Metrics(kc);
     const podsResponse = await coreApi
-      .listNamespacedPod(namespace, null, null, null, null, labelSelector)
+      .listNamespacedPod({
+        namespace,
+        labelSelector,
+      })
       .then((response) => response.body)
       .catch((err) => {
         logger.error("Error getting pods: %d %s %s", err.statusCode, err.body, err.response);

src/pages/api/kubernetes/status/[...service].js

@@ -1,6 +1,6 @@
 import { CoreV1Api } from "@kubernetes/client-node";
 
-import getKubeConfig from "../../../../utils/config/kubernetes";
+import { getKubeConfig } from "../../../../utils/config/kubernetes";
 import createLogger from "../../../../utils/logger";
 
 const logger = createLogger("kubernetesStatusService");
@@ -27,8 +27,11 @@ export default async function handler(req, res) {
     }
     const coreApi = kc.makeApiClient(CoreV1Api);
     const podsResponse = await coreApi
-      .listNamespacedPod(namespace, null, null, null, null, labelSelector)
-      .then((response) => response.body)
+      .listNamespacedPod({
+        namespace,
+        labelSelector,
+      })
+      .then((response) => response)
       .catch((err) => {
         logger.error("Error getting pods: %d %s %s", err.statusCode, err.body, err.response);
         return null;

src/pages/api/widgets/kubernetes.js

@@ -1,10 +1,10 @@
 import { CoreV1Api, Metrics } from "@kubernetes/client-node";
 
-import getKubeConfig from "../../../utils/config/kubernetes";
-import { parseCpu, parseMemory } from "../../../utils/kubernetes/kubernetes-utils";
+import { getKubeConfig } from "../../../utils/config/kubernetes";
+import { parseCpu, parseMemory } from "../../../utils/kubernetes/utils";
 import createLogger from "../../../utils/logger";
 
-const logger = createLogger("kubernetes-widget");
+const logger = createLogger("widget");
 
 export default async function handler(req, res) {
   try {

src/utils/config/kubernetes.js

@@ -2,18 +2,21 @@ import path from "path";
 import { readFileSync } from "fs";
 
 import yaml from "js-yaml";
-import { KubeConfig } from "@kubernetes/client-node";
+import { KubeConfig, ApiextensionsV1Api } from "@kubernetes/client-node";
 
 import checkAndCopyConfig, { CONF_DIR, substituteEnvironmentVars } from "utils/config/config";
 
-export default function getKubeConfig() {
+export function getKubernetes() {
   checkAndCopyConfig("kubernetes.yaml");
-
   const configFile = path.join(CONF_DIR, "kubernetes.yaml");
   const rawConfigData = readFileSync(configFile, "utf8");
   const configData = substituteEnvironmentVars(rawConfigData);
-  const config = yaml.load(configData);
+  return yaml.load(configData);
+}
+
+export const getKubeConfig = () => {
   const kc = new KubeConfig();
+  const config = getKubernetes();
 
   switch (config?.mode) {
     case "cluster":
@@ -28,4 +31,31 @@ export default function getKubeConfig() {
   }
 
   return kc;
+};
+
+export async function checkCRD(name, kc, logger) {
+  const apiExtensions = kc.makeApiClient(ApiextensionsV1Api);
+  const exist = await apiExtensions
+    .readCustomResourceDefinitionStatus({
+      name,
+    })
+    .then(() => true)
+    .catch(async (error) => {
+      if (error.statusCode === 403) {
+        logger.error(
+          "Error checking if CRD %s exists. Make sure to add the following permission to your RBAC: %d %s %s",
+          name,
+          error.statusCode,
+          error.body.message,
+        );
+      }
+      return false;
+    });
+
+  return exist;
 }
+
+export const ANNOTATION_BASE = "gethomepage.dev";
+export const ANNOTATION_WIDGET_BASE = `${ANNOTATION_BASE}/widget.`;
+export const HTTPROUTE_API_GROUP = "gateway.networking.k8s.io";
+export const HTTPROUTE_API_VERSION = "v1";

src/utils/config/service-helpers.js

@@ -3,12 +3,12 @@ import path from "path";
 
 import yaml from "js-yaml";
 import Docker from "dockerode";
-import { CustomObjectsApi, NetworkingV1Api, ApiextensionsV1Api } from "@kubernetes/client-node";
 
 import createLogger from "utils/logger";
 import checkAndCopyConfig, { CONF_DIR, getSettings, substituteEnvironmentVars } from "utils/config/config";
 import getDockerArguments from "utils/config/docker";
-import getKubeConfig from "utils/config/kubernetes";
+import kubernetes from "utils/kubernetes/export";
+import { getKubeConfig } from "utils/config/kubernetes";
 import * as shvl from "utils/config/shvl";
 
 const logger = createLogger("service-helpers");
@@ -167,36 +167,7 @@ export async function servicesFromDocker() {
   return mappedServiceGroups;
 }
 
-function getUrlFromIngress(ingress) {
-  const urlHost = ingress.spec.rules[0].host;
-  const urlPath = ingress.spec.rules[0].http.paths[0].path;
-  const urlSchema = ingress.spec.tls ? "https" : "http";
-  return `${urlSchema}://${urlHost}${urlPath}`;
-}
-
-export async function checkCRD(kc, name) {
-  const apiExtensions = kc.makeApiClient(ApiextensionsV1Api);
-  const exist = await apiExtensions
-    .readCustomResourceDefinitionStatus(name)
-    .then(() => true)
-    .catch(async (error) => {
-      if (error.statusCode === 403) {
-        logger.error(
-          "Error checking if CRD %s exists. Make sure to add the following permission to your RBAC: %d %s %s",
-          name,
-          error.statusCode,
-          error.body.message,
-        );
-      }
-      return false;
-    });
-
-  return exist;
-}
-
 export async function servicesFromKubernetes() {
-  const ANNOTATION_BASE = "gethomepage.dev";
-  const ANNOTATION_WIDGET_BASE = `${ANNOTATION_BASE}/widget.`;
   const { instanceName } = getSettings();
 
   checkAndCopyConfig("kubernetes.yaml");
@@ -206,145 +177,46 @@ export async function servicesFromKubernetes() {
     if (!kc) {
       return [];
     }
-    const networking = kc.makeApiClient(NetworkingV1Api);
-    const crd = kc.makeApiClient(CustomObjectsApi);
 
-    const ingressList = await networking
-      .listIngressForAllNamespaces(null, null, null, null)
-      .then((response) => response.body)
-      .catch((error) => {
-        logger.error("Error getting ingresses: %d %s %s", error.statusCode, error.body, error.response);
-        if (error) logger.debug(error);
-        return null;
-      });
+    // resource lists
+    const [ingressList, traefikIngressList, httpRouteList] = await Promise.all([
+      kubernetes.listIngress(),
+      kubernetes.listTraefikIngress(),
+      kubernetes.listHttpRoute(),
+    ]);
 
-    const traefikContainoExists = await checkCRD(kc, "ingressroutes.traefik.containo.us");
-    const traefikExists = await checkCRD(kc, "ingressroutes.traefik.io");
+    const resources = [...ingressList, ...traefikIngressList, ...httpRouteList];
 
-    const traefikIngressListContaino = await crd
-      .listClusterCustomObject("traefik.containo.us", "v1alpha1", "ingressroutes")
-      .then((response) => response.body)
-      .catch(async (error) => {
-        if (traefikContainoExists) {
-          logger.error(
-            "Error getting traefik ingresses from traefik.containo.us: %d %s %s",
-            error.statusCode,
-            error.body,
-            error.response,
-          );
-          if (error) logger.debug(error);
-        }
-
-        return [];
-      });
-
-    const traefikIngressListIo = await crd
-      .listClusterCustomObject("traefik.io", "v1alpha1", "ingressroutes")
-      .then((response) => response.body)
-      .catch(async (error) => {
-        if (traefikExists) {
-          logger.error(
-            "Error getting traefik ingresses from traefik.io: %d %s %s",
-            error.statusCode,
-            error.body,
-            error.response,
-          );
-          if (error) logger.debug(error);
-        }
-
-        return [];
-      });
-
-    const traefikIngressList = [...(traefikIngressListContaino?.items ?? []), ...(traefikIngressListIo?.items ?? [])];
-
-    if (traefikIngressList.length > 0) {
-      const traefikServices = traefikIngressList.filter(
-        (ingress) => ingress.metadata.annotations && ingress.metadata.annotations[`${ANNOTATION_BASE}/href`],
-      );
-      ingressList.items.push(...traefikServices);
-    }
-
-    if (!ingressList) {
+    if (!resources) {
       return [];
     }
-    const services = ingressList.items
-      .filter(
-        (ingress) =>
-          ingress.metadata.annotations &&
-          ingress.metadata.annotations[`${ANNOTATION_BASE}/enabled`] === "true" &&
-          (!ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] ||
-            ingress.metadata.annotations[`${ANNOTATION_BASE}/instance`] === instanceName ||
-            `${ANNOTATION_BASE}/instance.${instanceName}` in ingress.metadata.annotations),
-      )
-      .map((ingress) => {
-        let constructedService = {
-          app: ingress.metadata.annotations[`${ANNOTATION_BASE}/app`] || ingress.metadata.name,
-          namespace: ingress.metadata.namespace,
-          href: ingress.metadata.annotations[`${ANNOTATION_BASE}/href`] || getUrlFromIngress(ingress),
-          name: ingress.metadata.annotations[`${ANNOTATION_BASE}/name`] || ingress.metadata.name,
-          group: ingress.metadata.annotations[`${ANNOTATION_BASE}/group`] || "Kubernetes",
-          weight: ingress.metadata.annotations[`${ANNOTATION_BASE}/weight`] || "0",
-          icon: ingress.metadata.annotations[`${ANNOTATION_BASE}/icon`] || "",
-          description: ingress.metadata.annotations[`${ANNOTATION_BASE}/description`] || "",
-          external: false,
-          type: "service",
-        };
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]) {
-          constructedService.external =
-            String(ingress.metadata.annotations[`${ANNOTATION_BASE}/external`]).toLowerCase() === "true";
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`] !== undefined) {
-          constructedService.podSelector = ingress.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`];
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`]) {
-          constructedService.ping = ingress.metadata.annotations[`${ANNOTATION_BASE}/ping`];
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`]) {
-          constructedService.siteMonitor = ingress.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`];
-        }
-        if (ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`]) {
-          constructedService.statusStyle = ingress.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`];
-        }
-        Object.keys(ingress.metadata.annotations).forEach((annotation) => {
-          if (annotation.startsWith(ANNOTATION_WIDGET_BASE)) {
-            shvl.set(
-              constructedService,
-              annotation.replace(`${ANNOTATION_BASE}/`, ""),
-              ingress.metadata.annotations[annotation],
-            );
-          }
-        });
+    const services = await Promise.all(
+      resources
+        .filter((resource) => kubernetes.isDiscoverable(resource, instanceName))
+        .map(async (resource) => kubernetes.constructedServiceFromResource(resource)),
+    );
 
-        try {
-          constructedService = JSON.parse(substituteEnvironmentVars(JSON.stringify(constructedService)));
-        } catch (e) {
-          logger.error("Error attempting k8s environment variable substitution.");
-          if (e) logger.debug(e);
-        }
+    // map service groups
+    const mappedServiceGroups = services.reduce((groups, serverService) => {
+      let serverGroup = groups.find((group) => group.name === serverService.group);
 
-        return constructedService;
-      });
-
-    const mappedServiceGroups = [];
-
-    services.forEach((serverService) => {
-      let serverGroup = mappedServiceGroups.find((searchedGroup) => searchedGroup.name === serverService.group);
       if (!serverGroup) {
-        mappedServiceGroups.push({
+        serverGroup = {
           name: serverService.group,
           services: [],
-        });
-        serverGroup = mappedServiceGroups[mappedServiceGroups.length - 1];
+        };
+        groups.push(serverGroup);
       }
 
-      const { name: serviceName, group: serverServiceGroup, ...pushedService } = serverService;
-      const result = {
+      const { name: serviceName, group: _, ...pushedService } = serverService;
+
+      serverGroup.services.push({
         name: serviceName,
         ...pushedService,
-      };
+      });
 
-      serverGroup.services.push(result);
-    });
+      return groups;
+    }, []);
 
     return mappedServiceGroups;
   } catch (e) {

src/utils/kubernetes/export.js

@@ -0,0 +1,14 @@
+import listIngress from "utils/kubernetes/ingress-list";
+import listTraefikIngress from "utils/kubernetes/traefik-list";
+import listHttpRoute from "utils/kubernetes/httproute-list";
+import { isDiscoverable, constructedServiceFromResource } from "utils/kubernetes/resource-helpers";
+
+const kubernetes = {
+  listIngress,
+  listTraefikIngress,
+  listHttpRoute,
+  isDiscoverable,
+  constructedServiceFromResource,
+};
+
+export default kubernetes;

src/utils/kubernetes/httproute-list.js

@@ -0,0 +1,56 @@
+import { CustomObjectsApi, CoreV1Api } from "@kubernetes/client-node";
+
+import { getKubernetes, getKubeConfig, HTTPROUTE_API_GROUP, HTTPROUTE_API_VERSION } from "utils/config/kubernetes";
+import createLogger from "utils/logger";
+
+const logger = createLogger("httproute-list");
+const kc = getKubeConfig();
+
+export default async function listHttpRoute() {
+  const crd = kc.makeApiClient(CustomObjectsApi);
+  const core = kc.makeApiClient(CoreV1Api);
+  const { gateway } = getKubernetes();
+  let httpRouteList = [];
+
+  if (gateway) {
+    // httproutes
+    const getHttpRoute = async (namespace) =>
+      crd
+        .listNamespacedCustomObject({
+          group: HTTPROUTE_API_GROUP,
+          version: HTTPROUTE_API_VERSION,
+          namespace,
+          plural: "httproutes",
+        })
+        .then((response) => {
+          const [httpRoute] = response.items;
+          return httpRoute;
+        })
+        .catch((error) => {
+          logger.error("Error getting httproutes: %d %s %s", error.statusCode, error.body, error.response);
+          logger.debug(error);
+          return null;
+        });
+    // namespaces
+    const namespaces = await core
+      .listNamespace()
+      .then((response) => response.items.map((ns) => ns.metadata.name))
+      .catch((error) => {
+        logger.error("Error getting namespaces: %d %s %s", error.statusCode, error.body, error.response);
+        logger.debug(error);
+        return null;
+      });
+
+    if (namespaces) {
+      const httpRouteListUnfiltered = await Promise.all(
+        namespaces.map(async (namespace) => {
+          const httpRoute = await getHttpRoute(namespace);
+          return httpRoute;
+        }),
+      );
+
+      httpRouteList = httpRouteListUnfiltered.filter((httpRoute) => httpRoute !== undefined);
+    }
+  }
+  return httpRouteList;
+}

src/utils/kubernetes/ingress-list.js

@@ -0,0 +1,26 @@
+import { NetworkingV1Api } from "@kubernetes/client-node";
+
+import { getKubernetes, getKubeConfig } from "utils/config/kubernetes";
+import createLogger from "utils/logger";
+
+const logger = createLogger("ingress-list");
+const kc = getKubeConfig();
+
+export default async function listIngress() {
+  const networking = kc.makeApiClient(NetworkingV1Api);
+  const { ingress } = getKubernetes();
+  let ingressList = [];
+
+  if (ingress) {
+    const ingressData = await networking
+      .listIngressForAllNamespaces()
+      .then((response) => response)
+      .catch((error) => {
+        logger.error("Error getting ingresses: %d %s %s", error.statusCode, error.body, error.response);
+        logger.debug(error);
+        return null;
+      });
+    ingressList = ingressData.items;
+  }
+  return ingressList;
+}

src/utils/kubernetes/resource-helpers.js

@@ -0,0 +1,130 @@
+import { CustomObjectsApi } from "@kubernetes/client-node";
+
+import {
+  getKubeConfig,
+  ANNOTATION_BASE,
+  ANNOTATION_WIDGET_BASE,
+  HTTPROUTE_API_GROUP,
+  HTTPROUTE_API_VERSION,
+} from "utils/config/kubernetes";
+import { substituteEnvironmentVars } from "utils/config/config";
+import createLogger from "utils/logger";
+import * as shvl from "utils/config/shvl";
+
+const logger = createLogger("resource-helpers");
+const kc = getKubeConfig();
+
+const getSchemaFromGateway = async (gatewayRef) => {
+  const crd = kc.makeApiClient(CustomObjectsApi);
+  const schema = await crd
+    .getNamespacedCustomObject({
+      group: HTTPROUTE_API_GROUP,
+      version: HTTPROUTE_API_VERSION,
+      namespace: gatewayRef.namespace,
+      plural: "gateways",
+      name: gatewayRef.name,
+    })
+    .then((response) => {
+      const listner = response.spec.listeners.filter((listener) => listener.name === gatewayRef.sectionName)[0];
+      return listner.protocol.toLowerCase();
+    })
+    .catch((error) => {
+      logger.error("Error getting gateways: %d %s %s", error.statusCode, error.body, error.response);
+      logger.debug(error);
+      return "";
+    });
+  return schema;
+};
+
+async function getUrlFromHttpRoute(resource) {
+  let url = null;
+  const hasHostName = resource.spec?.hostnames;
+
+  if (hasHostName) {
+    if (resource.spec.rules[0].matches[0].path.type !== "RegularExpression") {
+      const urlHost = resource.spec.hostnames[0];
+      const urlPath = resource.spec.rules[0].matches[0].path.value;
+      const urlSchema = (await getSchemaFromGateway(resource.spec.parentRefs[0])) ? "https" : "http";
+      url = `${urlSchema}://${urlHost}${urlPath}`;
+    }
+  }
+  return url;
+}
+
+function getUrlFromIngress(resource) {
+  const urlHost = resource.spec.rules[0].host;
+  const urlPath = resource.spec.rules[0].http.paths[0].path;
+  const urlSchema = resource.spec.tls ? "https" : "http";
+  return `${urlSchema}://${urlHost}${urlPath}`;
+}
+
+async function getUrlSchema(resource) {
+  const isHttpRoute = resource.kind === "HTTPRoute";
+  let urlSchema;
+  if (isHttpRoute) {
+    urlSchema = getUrlFromHttpRoute(resource);
+  } else {
+    urlSchema = getUrlFromIngress(resource);
+  }
+  return urlSchema;
+}
+
+export function isDiscoverable(resource, instanceName) {
+  return (
+    resource.metadata.annotations &&
+    resource.metadata.annotations[`${ANNOTATION_BASE}/enabled`] === "true" &&
+    (!resource.metadata.annotations[`${ANNOTATION_BASE}/instance`] ||
+      resource.metadata.annotations[`${ANNOTATION_BASE}/instance`] === instanceName ||
+      `${ANNOTATION_BASE}/instance.${instanceName}` in resource.metadata.annotations)
+  );
+}
+
+export async function constructedServiceFromResource(resource) {
+  let constructedService = {
+    app: resource.metadata.annotations[`${ANNOTATION_BASE}/app`] || resource.metadata.name,
+    namespace: resource.metadata.namespace,
+    href: resource.metadata.annotations[`${ANNOTATION_BASE}/href`] || (await getUrlSchema(resource)),
+    name: resource.metadata.annotations[`${ANNOTATION_BASE}/name`] || resource.metadata.name,
+    group: resource.metadata.annotations[`${ANNOTATION_BASE}/group`] || "Kubernetes",
+    weight: resource.metadata.annotations[`${ANNOTATION_BASE}/weight`] || "0",
+    icon: resource.metadata.annotations[`${ANNOTATION_BASE}/icon`] || "",
+    description: resource.metadata.annotations[`${ANNOTATION_BASE}/description`] || "",
+    external: false,
+    type: "service",
+  };
+  if (resource.metadata.annotations[`${ANNOTATION_BASE}/external`]) {
+    constructedService.external =
+      String(resource.metadata.annotations[`${ANNOTATION_BASE}/external`]).toLowerCase() === "true";
+  }
+  if (resource.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`] !== undefined) {
+    constructedService.podSelector = resource.metadata.annotations[`${ANNOTATION_BASE}/pod-selector`];
+  }
+  if (resource.metadata.annotations[`${ANNOTATION_BASE}/ping`]) {
+    constructedService.ping = resource.metadata.annotations[`${ANNOTATION_BASE}/ping`];
+  }
+  if (resource.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`]) {
+    constructedService.siteMonitor = resource.metadata.annotations[`${ANNOTATION_BASE}/siteMonitor`];
+  }
+  if (resource.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`]) {
+    constructedService.statusStyle = resource.metadata.annotations[`${ANNOTATION_BASE}/statusStyle`];
+  }
+
+  Object.keys(resource.metadata.annotations).forEach((annotation) => {
+    if (annotation.startsWith(ANNOTATION_WIDGET_BASE)) {
+      shvl.set(
+        constructedService,
+        annotation.replace(`${ANNOTATION_BASE}/`, ""),
+        resource.metadata.annotations[annotation],
+      );
+    }
+  });
+
+  try {
+    constructedService = JSON.parse(substituteEnvironmentVars(JSON.stringify(constructedService)));
+  } catch (e) {
+    logger.error("Error attempting k8s environment variable substitution.");
+    logger.debug(e);
+  }
+
+  return constructedService;
+}

src/utils/kubernetes/traefik-list.js

@@ -0,0 +1,70 @@
+import { CustomObjectsApi } from "@kubernetes/client-node";
+
+import { getKubernetes, getKubeConfig, checkCRD, ANNOTATION_BASE } from "utils/config/kubernetes";
+import createLogger from "utils/logger";
+
+const logger = createLogger("traefik-list");
+const kc = getKubeConfig();
+
+export default async function listTraefikIngress() {
+  const { traefik } = getKubernetes();
+  const traefikList = [];
+
+  if (traefik) {
+    const crd = kc.makeApiClient(CustomObjectsApi);
+    const traefikContainoExists = await checkCRD("ingressroutes.traefik.containo.us", kc, logger);
+    const traefikExists = await checkCRD("ingressroutes.traefik.io", kc, logger);
+
+    const traefikIngressListContaino = await crd
+      .listClusterCustomObject({
+        group: "traefik.containo.us",
+        version: "v1alpha1",
+        plural: "ingressroutes",
+      })
+      .then((response) => response)
+      .catch(async (error) => {
+        if (traefikContainoExists) {
+          logger.error(
+            "Error getting traefik ingresses from traefik.containo.us: %d %s %s",
+            error.statusCode,
+            error.body,
+            error.response,
+          );
+          logger.debug(error);
+        }
+
+        return [];
+      });
+
+    const traefikIngressListIo = await crd
+      .listClusterCustomObject({
+        group: "traefik.io",
+        version: "v1alpha1",
+        plural: "ingressroutes",
+      })
+      .then((response) => response.body)
+      .catch(async (error) => {
+        if (traefikExists) {
+          logger.error(
+            "Error getting traefik ingresses from traefik.io: %d %s %s",
+            error.statusCode,
+            error.body,
+            error.response,
+          );
+          logger.debug(error);
+        }
+
+        return [];
+      });
+
+    const traefikIngressList = [...(traefikIngressListContaino?.items ?? []), ...(traefikIngressListIo?.items ?? [])];
+
+    if (traefikIngressList.length > 0) {
+      const traefikServices = traefikIngressList.filter(
+        (ingress) => ingress.metadata.annotations && ingress.metadata.annotations[`${ANNOTATION_BASE}/href`],
+      );
+      traefikList.push(...traefikServices);
+    }
+  }
+  return traefikList;
+}