From c4333fd2dc6a87a08b747f96d4e075a34c858cec Mon Sep 17 00:00:00 2001 From: James Wynn Date: Mon, 24 Oct 2022 17:03:35 -0500 Subject: [PATCH] Kubernetes support * Total CPU and Memory usage for the entire cluster * Total CPU and Memory usage for kubernetes pods * Service discovery via annotations on ingress * No storage stats yet * No network stats yet --- .gitignore | 3 + package.json | 1 + src/components/services/item.jsx | 22 ++++++ src/components/services/kubernetes-status.jsx | 19 +++++ src/components/widgets/resources/cpu.jsx | 4 +- src/components/widgets/resources/disk.jsx | 4 +- src/components/widgets/resources/memory.jsx | 4 +- .../widgets/resources/resources.jsx | 10 +-- .../api/kubernetes/stats/[...service].js | 79 +++++++++++++++++++ .../api/kubernetes/status/[...service].js | 42 ++++++++++ src/pages/api/widgets/kubernetes.js | 72 +++++++++++++++++ src/skeleton/kubernetes.yaml | 2 + src/utils/config/api-response.js | 37 +++++++-- src/utils/config/kubernetes.js | 27 +++++++ src/utils/config/service-helpers.js | 70 +++++++++++++++- src/utils/kubernetes/kubernetes-utils.js | 47 +++++++++++ src/widgets/components.js | 1 + src/widgets/kubernetes/component.jsx | 54 +++++++++++++ 18 files changed, 479 insertions(+), 19 deletions(-) create mode 100644 src/components/services/kubernetes-status.jsx create mode 100644 src/pages/api/kubernetes/stats/[...service].js create mode 100644 src/pages/api/kubernetes/status/[...service].js create mode 100644 src/pages/api/widgets/kubernetes.js create mode 100644 src/skeleton/kubernetes.yaml create mode 100644 src/utils/config/kubernetes.js create mode 100644 src/utils/kubernetes/kubernetes-utils.js create mode 100644 src/widgets/kubernetes/component.jsx diff --git a/.gitignore b/.gitignore index 7ab221f97..9bd31081c 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,6 @@ next-env.d.ts # homepage /config + +# idea +.idea/ diff --git a/package.json b/package.json index 75d02e7fd..2dddf7250 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@headlessui/react": "^1.7.2", + "@kubernetes/client-node": "^0.17.1", "classnames": "^2.3.2", "compare-versions": "^5.0.1", "dockerode": "^3.3.4", diff --git a/src/components/services/item.jsx b/src/components/services/item.jsx index 56ed2b4b1..be6cf6c52 100644 --- a/src/components/services/item.jsx +++ b/src/components/services/item.jsx @@ -3,8 +3,10 @@ import { useContext, useState } from "react"; import Status from "./status"; import Widget from "./widget"; +import KubernetesStatus from "./kubernetes-status"; import Docker from "widgets/docker/component"; +import Kubernetes from "widgets/kubernetes/component"; import { SettingsContext } from "utils/contexts/settings"; import ResolvedIcon from "components/resolvedicon"; @@ -80,6 +82,16 @@ export default function Item({ service }) { View container stats )} + {service.app && ( + + )} {service.container && service.server && ( @@ -92,6 +104,16 @@ export default function Item({ service }) { {statsOpen && } )} + {service.app && ( +
+ {statsOpen && } +
+ )} {service.widget && } diff --git a/src/components/services/kubernetes-status.jsx b/src/components/services/kubernetes-status.jsx new file mode 100644 index 000000000..4b2c57236 --- /dev/null +++ b/src/components/services/kubernetes-status.jsx @@ -0,0 +1,19 @@ +import useSWR from "swr"; + +export default function KubernetesStatus({ service }) { + const { data, error } = useSWR(`/api/kubernetes/status/${service.namespace}/${service.app}`); + + if (error) { + return
; + } + + if (data && data.status === "running") { + return
; + } + + if (data && data.status === "not found") { + return
; + } + + return
; +} diff --git a/src/components/widgets/resources/cpu.jsx b/src/components/widgets/resources/cpu.jsx index 6b021193b..9564d99a1 100644 --- a/src/components/widgets/resources/cpu.jsx +++ b/src/components/widgets/resources/cpu.jsx @@ -5,10 +5,10 @@ import { useTranslation } from "next-i18next"; import UsageBar from "./usage-bar"; -export default function Cpu({ expanded }) { +export default function Cpu({ expanded, backend }) { const { t } = useTranslation(); - const { data, error } = useSWR(`/api/widgets/resources?type=cpu`, { + const { data, error } = useSWR(`/api/widgets/${backend || 'resources'}?type=cpu`, { refreshInterval: 1500, }); diff --git a/src/components/widgets/resources/disk.jsx b/src/components/widgets/resources/disk.jsx index 69f560f62..eea84162e 100644 --- a/src/components/widgets/resources/disk.jsx +++ b/src/components/widgets/resources/disk.jsx @@ -5,10 +5,10 @@ import { useTranslation } from "next-i18next"; import UsageBar from "./usage-bar"; -export default function Disk({ options, expanded }) { +export default function Disk({ options, expanded, backend }) { const { t } = useTranslation(); - const { data, error } = useSWR(`/api/widgets/resources?type=disk&target=${options.disk}`, { + const { data, error } = useSWR(`/api/widgets/${backend || 'resources'}?type=disk&target=${options.disk}`, { refreshInterval: 1500, }); diff --git a/src/components/widgets/resources/memory.jsx b/src/components/widgets/resources/memory.jsx index 452634565..db5be4b9b 100644 --- a/src/components/widgets/resources/memory.jsx +++ b/src/components/widgets/resources/memory.jsx @@ -5,10 +5,10 @@ import { useTranslation } from "next-i18next"; import UsageBar from "./usage-bar"; -export default function Memory({ expanded }) { +export default function Memory({ expanded, backend }) { const { t } = useTranslation(); - const { data, error } = useSWR(`/api/widgets/resources?type=memory`, { + const { data, error } = useSWR(`/api/widgets/${backend || 'resources'}?type=memory`, { refreshInterval: 1500, }); diff --git a/src/components/widgets/resources/resources.jsx b/src/components/widgets/resources/resources.jsx index 0524e39af..c4072c7a1 100644 --- a/src/components/widgets/resources/resources.jsx +++ b/src/components/widgets/resources/resources.jsx @@ -3,15 +3,15 @@ import Cpu from "./cpu"; import Memory from "./memory"; export default function Resources({ options }) { - const { expanded } = options; + const { expanded, backend } = options; return (
- {options.cpu && } - {options.memory && } + {options.cpu && } + {options.memory && } {Array.isArray(options.disk) - ? options.disk.map((disk) => ) - : options.disk && } + ? options.disk.map((disk) => ) + : options.disk && }
{options.label && (
{options.label}
diff --git a/src/pages/api/kubernetes/stats/[...service].js b/src/pages/api/kubernetes/stats/[...service].js new file mode 100644 index 000000000..05001908d --- /dev/null +++ b/src/pages/api/kubernetes/stats/[...service].js @@ -0,0 +1,79 @@ +import { CoreV1Api, Metrics } from "@kubernetes/client-node"; + +import getKubeConfig from "../../../../utils/config/kubernetes"; +import { parseCpu, parseMemory } from "../../../../utils/kubernetes/kubernetes-utils"; + +export default async function handler(req, res) { + const APP_LABEL = "app.kubernetes.io/name"; + const { service } = req.query; + + const [namespace, appName] = service; + if (!namespace && !appName) { + res.status(400).send({ + error: "kubernetes query parameters are required", + }); + return; + } + const labelSelector = `${APP_LABEL}=${appName}`; + + try { + const kc = getKubeConfig(); + const coreApi = kc.makeApiClient(CoreV1Api); + const metricsApi = new Metrics(kc); + const podsResponse = await coreApi.listNamespacedPod(namespace, null, null, null, null, labelSelector); + const pods = podsResponse.body.items; + + if (pods.length === 0) { + res.status(200).send({ + error: "not found", + }); + return; + } + + let cpuLimit = 0; + let memLimit = 0; + pods.forEach((pod) => { + pod.spec.containers.forEach((container) => { + if (container?.resources?.limits?.cpu) { + cpuLimit += parseCpu(container?.resources?.limits?.cpu); + } + if (container?.resources?.limits?.memory) { + memLimit += parseMemory(container?.resources?.limits?.memory); + } + }); + }); + + const stats = await pods.map(async (pod) => { + let depMem = 0; + let depCpu = 0; + const podMetrics = await metricsApi.getPodMetrics(namespace, pod.metadata.name); + podMetrics.containers.forEach((container) => { + depMem += parseMemory(container.usage.memory); + depCpu += parseCpu(container.usage.cpu); + }); + return { + mem: depMem, + cpu: depCpu + } + }).reduce(async (finalStats, podStatPromise) => { + const podStats = await podStatPromise; + return { + mem: finalStats.mem + podStats.mem, + cpu: finalStats.cpu + podStats.cpu + }; + }); + stats.cpuLimit = cpuLimit; + stats.memLimit = memLimit; + stats.cpuUsage = stats.cpu / cpuLimit; + stats.memUsage = stats.mem / memLimit; + + res.status(200).json({ + stats, + }); + } catch (e) { + console.log("error", e); + res.status(500).send({ + error: "unknown error", + }); + } +} diff --git a/src/pages/api/kubernetes/status/[...service].js b/src/pages/api/kubernetes/status/[...service].js new file mode 100644 index 000000000..dbe64f38b --- /dev/null +++ b/src/pages/api/kubernetes/status/[...service].js @@ -0,0 +1,42 @@ +import { CoreV1Api } from "@kubernetes/client-node"; + +import getKubeConfig from "../../../../utils/config/kubernetes"; + +export default async function handler(req, res) { + const APP_LABEL = "app.kubernetes.io/name"; + const { service } = req.query; + + const [namespace, appName] = service; + if (!namespace && !appName) { + res.status(400).send({ + error: "kubernetes query parameters are required", + }); + return; + } + const labelSelector = `${APP_LABEL}=${appName}`; + + try { + const kc = getKubeConfig(); + const coreApi = kc.makeApiClient(CoreV1Api); + const podsResponse = await coreApi.listNamespacedPod(namespace, null, null, null, null, labelSelector); + const pods = podsResponse.body.items; + + if (pods.length === 0) { + res.status(200).send({ + error: "not found", + }); + return; + } + + // at least one pod must be in the "Running" phase, otherwise its "down" + const runningPod = pods.find(pod => pod.status.phase === "Running"); + const status = runningPod ? "running" : "down"; + res.status(200).json({ + status + }); + } catch { + res.status(500).send({ + error: "unknown error", + }); + } +} diff --git a/src/pages/api/widgets/kubernetes.js b/src/pages/api/widgets/kubernetes.js new file mode 100644 index 000000000..a740df903 --- /dev/null +++ b/src/pages/api/widgets/kubernetes.js @@ -0,0 +1,72 @@ +import { CoreV1Api, Metrics } from "@kubernetes/client-node"; + +import getKubeConfig from "../../../utils/config/kubernetes"; +import { parseCpu, parseMemory } from "../../../utils/kubernetes/kubernetes-utils"; + +export default async function handler(req, res) { + const { type } = req.query; + + const kc = getKubeConfig(); + const coreApi = kc.makeApiClient(CoreV1Api); + const metricsApi = new Metrics(kc); + + const nodes = await coreApi.listNode(); + const nodeCapacity = new Map(); + let cpuTotal = 0; + let cpuUsage = 0; + let memTotal = 0; + let memUsage = 0; + + nodes.body.items.forEach((node) => { + nodeCapacity.set(node.metadata.name, node.status.capacity); + cpuTotal += Number.parseInt(node.status.capacity.cpu, 10); + memTotal += parseMemory(node.status.capacity.memory); + }); + + const nodeMetrics = await metricsApi.getNodeMetrics(); + const nodeUsage = new Map(); + nodeMetrics.items.forEach((metrics) => { + nodeUsage.set(metrics.metadata.name, metrics.usage); + cpuUsage += parseCpu(metrics.usage.cpu); + memUsage += parseMemory(metrics.usage.memory); + }); + + if (type === "cpu") { + return res.status(200).json({ + cpu: { + usage: (cpuUsage / cpuTotal) * 100, + load: cpuUsage + } + }); + } + // Maybe Storage CSI can provide this information + // if (type === "disk") { + // if (!existsSync(target)) { + // return res.status(404).json({ + // error: "Target not found", + // }); + // } + // + // return res.status(200).json({ + // drive: await drive.info(target || "/"), + // }); + // } + // + if (type === "memory") { + const SCALE_MB = 1024 * 1024; + const usedMemMb = memUsage / SCALE_MB; + const totalMemMb = memTotal / SCALE_MB; + const freeMemMb = totalMemMb - usedMemMb; + return res.status(200).json({ + memory: { + usedMemMb, + freeMemMb, + totalMemMb + } + }); + } + + return res.status(400).json({ + error: "invalid type" + }); +} diff --git a/src/skeleton/kubernetes.yaml b/src/skeleton/kubernetes.yaml new file mode 100644 index 000000000..aca6e8213 --- /dev/null +++ b/src/skeleton/kubernetes.yaml @@ -0,0 +1,2 @@ +--- +# sample kubernetes config diff --git a/src/utils/config/api-response.js b/src/utils/config/api-response.js index 5cc1127e3..aef1650c9 100644 --- a/src/utils/config/api-response.js +++ b/src/utils/config/api-response.js @@ -5,7 +5,12 @@ import path from "path"; import yaml from "js-yaml"; import checkAndCopyConfig from "utils/config/config"; -import { servicesFromConfig, servicesFromDocker, cleanServiceGroups } from "utils/config/service-helpers"; +import { + servicesFromConfig, + servicesFromDocker, + cleanServiceGroups, + servicesFromKubernetes +} from "utils/config/service-helpers"; import { cleanWidgetGroups, widgetsFromConfig } from "utils/config/widget-helpers"; export async function bookmarksResponse() { @@ -44,15 +49,24 @@ export async function widgetsResponse() { } export async function servicesResponse() { - let discoveredServices; + let discoveredDockerServices; + let discoveredKubernetesServices; let configuredServices; try { - discoveredServices = cleanServiceGroups(await servicesFromDocker()); + discoveredDockerServices = cleanServiceGroups(await servicesFromDocker()); } catch (e) { console.error("Failed to discover services, please check docker.yaml for errors or remove example entries."); if (e) console.error(e); - discoveredServices = []; + discoveredDockerServices = []; + } + + try { + discoveredKubernetesServices = cleanServiceGroups(await servicesFromKubernetes()); + } catch (e) { + console.error("Failed to discover services, please check docker.yaml for errors or remove example entries."); + if (e) console.error(e); + discoveredKubernetesServices = []; } try { @@ -64,18 +78,27 @@ export async function servicesResponse() { } const mergedGroupsNames = [ - ...new Set([discoveredServices.map((group) => group.name), configuredServices.map((group) => group.name)].flat()), + ...new Set([ + discoveredDockerServices.map((group) => group.name), + discoveredKubernetesServices.map((group) => group.name), + configuredServices.map((group) => group.name), + ].flat()), ]; const mergedGroups = []; mergedGroupsNames.forEach((groupName) => { - const discoveredGroup = discoveredServices.find((group) => group.name === groupName) || { services: [] }; + const discoveredDockerGroup = discoveredDockerServices.find((group) => group.name === groupName) || { services: [] }; + const discoveredKubernetesGroup = discoveredKubernetesServices.find((group) => group.name === groupName) || { services: [] }; const configuredGroup = configuredServices.find((group) => group.name === groupName) || { services: [] }; const mergedGroup = { name: groupName, - services: [...discoveredGroup.services, ...configuredGroup.services].filter((service) => service), + services: [ + ...discoveredDockerGroup.services, + ...discoveredKubernetesGroup.services, + ...configuredGroup.services + ].filter((service) => service), }; mergedGroups.push(mergedGroup); diff --git a/src/utils/config/kubernetes.js b/src/utils/config/kubernetes.js new file mode 100644 index 000000000..b6b1adc7f --- /dev/null +++ b/src/utils/config/kubernetes.js @@ -0,0 +1,27 @@ +import path from "path"; +import { readFileSync } from "fs"; + +import yaml from "js-yaml"; +import { KubeConfig } from "@kubernetes/client-node"; + +import checkAndCopyConfig from "utils/config/config"; + +export default function getKubeConfig() { + checkAndCopyConfig("kubernetes.yaml"); + + const configFile = path.join(process.cwd(), "config", "kubernetes.yaml"); + const configData = readFileSync(configFile, "utf8"); + const config = yaml.load(configData); + const kc = new KubeConfig(); + + switch (config?.mode) { + case 'cluster': + kc.loadFromCluster(); + break; + case 'default': + default: + kc.loadFromDefault(); + } + + return kc; +} diff --git a/src/utils/config/service-helpers.js b/src/utils/config/service-helpers.js index 15740d22b..b1a368eee 100644 --- a/src/utils/config/service-helpers.js +++ b/src/utils/config/service-helpers.js @@ -4,9 +4,11 @@ import path from "path"; import yaml from "js-yaml"; import Docker from "dockerode"; import * as shvl from "shvl"; +import { NetworkingV1Api } from "@kubernetes/client-node"; import checkAndCopyConfig from "utils/config/config"; import getDockerArguments from "utils/config/docker"; +import getKubeConfig from "utils/config/kubernetes"; export async function servicesFromConfig() { checkAndCopyConfig("services.yaml"); @@ -103,6 +105,56 @@ export async function servicesFromDocker() { return mappedServiceGroups; } +export async function servicesFromKubernetes() { + checkAndCopyConfig("kubernetes.yaml"); + + const kc = getKubeConfig(); + const networking = kc.makeApiClient(NetworkingV1Api); + + const ingressResponse = await networking.listIngressForAllNamespaces(null, null, null, "homepage/enabled=true"); + const services = ingressResponse.body.items.map((ingress) => { + const constructedService = { + app: ingress.metadata.name, + namespace: ingress.metadata.namespace, + href: `https://${ingress.spec.rules[0].host}`, + name: ingress.metadata.annotations['homepage/name'], + group: ingress.metadata.annotations['homepage/group'], + icon: ingress.metadata.annotations['homepage/icon'], + description: ingress.metadata.annotations['homepage/description'] + }; + Object.keys(ingress.metadata.labels).forEach((label) => { + if (label.startsWith("homepage/widget/")) { + shvl.set(constructedService, label.replace("homepage/widget/", ""), ingress.metadata.labels[label]); + } + }); + + return constructedService; + }); + + const mappedServiceGroups = []; + + services.forEach((serverService) => { + let serverGroup = mappedServiceGroups.find((searchedGroup) => searchedGroup.name === serverService.group); + if (!serverGroup) { + mappedServiceGroups.push({ + name: serverService.group, + services: [], + }); + serverGroup = mappedServiceGroups[mappedServiceGroups.length - 1]; + } + + const { name: serviceName, group: serverServiceGroup, ...pushedService } = serverService; + const result = { + name: serviceName, + ...pushedService, + }; + + serverGroup.services.push(result); + }); + + return mappedServiceGroups; +} + export function cleanServiceGroups(groups) { return groups.map((serviceGroup) => ({ name: serviceGroup.name, @@ -118,6 +170,8 @@ export function cleanServiceGroups(groups) { container, currency, // coinmarketcap widget symbols, + namespace, // kubernetes widget + app } = cleanedService.widget; cleanedService.widget = { @@ -134,6 +188,10 @@ export function cleanServiceGroups(groups) { if (server) cleanedService.widget.server = server; if (container) cleanedService.widget.container = container; } + if (type === "kubernetes") { + if (namespace) cleanedService.widget.namespace = namespace; + if (app) cleanedService.widget.app = app; + } } return cleanedService; @@ -164,5 +222,15 @@ export default async function getServiceWidget(group, service) { } } + const kubernetesServices = await servicesFromKubernetes(); + const kubernetesServiceGroup = kubernetesServices.find((g) => g.name === group); + if (kubernetesServiceGroup) { + const kubernetesServiceEntry = kubernetesServiceGroup.services.find((s) => s.name === service); + if (kubernetesServiceEntry) { + const { widget } = kubernetesServiceEntry; + return widget; + } + } + return false; -} \ No newline at end of file +} diff --git a/src/utils/kubernetes/kubernetes-utils.js b/src/utils/kubernetes/kubernetes-utils.js new file mode 100644 index 000000000..08bd53d34 --- /dev/null +++ b/src/utils/kubernetes/kubernetes-utils.js @@ -0,0 +1,47 @@ +export function parseCpu(cpuStr) { + const unitLength = 1; + const base = Number.parseInt(cpuStr, 10); + const units = cpuStr.substring(cpuStr.length - unitLength); + // console.log(Number.isNaN(Number(units)), cpuStr, base, units); + if (Number.isNaN(Number(units))) { + switch (units) { + case 'n': + return base / 1000000000; + case 'u': + return base / 1000000; + case 'm': + return base / 1000; + default: + return base; + } + } else { + return Number.parseInt(cpuStr, 10); + } +} + +export function parseMemory(memStr) { + const unitLength = (memStr.substring(memStr.length - 1) === 'i' ? 2 : 1); + const base = Number.parseInt(memStr, 10); + const units = memStr.substring(memStr.length - unitLength); + // console.log(Number.isNaN(Number(units)), memStr, base, units); + if (Number.isNaN(Number(units))) { + switch (units) { + case 'Ki': + return base * 1000; + case 'K': + return base * 1024; + case 'Mi': + return base * 1000000; + case 'M': + return base * 1024 * 1024; + case 'Gi': + return base * 1000000000; + case 'G': + return base * 1024 * 1024 * 1024; + default: + return base; + } + } else { + return Number.parseInt(memStr, 10); + } +} diff --git a/src/widgets/components.js b/src/widgets/components.js index 33d09eac3..026f9dade 100644 --- a/src/widgets/components.js +++ b/src/widgets/components.js @@ -8,6 +8,7 @@ const components = { changedetectionio: dynamic(() => import("./changedetectionio/component")), coinmarketcap: dynamic(() => import("./coinmarketcap/component")), docker: dynamic(() => import("./docker/component")), + kubernetes: dynamic(() => import("./kubernetes/component")), emby: dynamic(() => import("./emby/component")), gotify: dynamic(() => import("./gotify/component")), homebridge: dynamic(() => import("./homebridge/component")), diff --git a/src/widgets/kubernetes/component.jsx b/src/widgets/kubernetes/component.jsx new file mode 100644 index 000000000..9ed7627d2 --- /dev/null +++ b/src/widgets/kubernetes/component.jsx @@ -0,0 +1,54 @@ +import useSWR from "swr"; +import { useTranslation } from "next-i18next"; + +import Container from "components/services/widget/container"; +import Block from "components/services/widget/block"; + +export default function Component({ service }) { + const { t } = useTranslation(); + + const { widget } = service; + + const { data: statusData, error: statusError } = useSWR( + `/api/kubernetes/status/${widget.namespace}/${widget.app}`); + + const { data: statsData, error: statsError } = useSWR( + `/api/kubernetes/stats/${widget.namespace}/${widget.app}`); + + if (statsError || statusError) { + return ; + } + + if (statusData && statusData.status !== "running") { + return ( + + + + ); + } + + if (!statsData || !statusData) { + return ( + + + + + + + ); + } + + const network = statsData.stats?.networks?.eth0 || statsData.stats?.networks?.network; + return ( + + + + {network && ( + <> + + + + )} + + ); +}