From e0f1aae4d5b847922ce743c621055007cb22854a Mon Sep 17 00:00:00 2001 From: James Wynn Date: Thu, 23 Feb 2023 08:50:25 -0600 Subject: [PATCH 1/2] Added support for environment variable substitution * Only environment variables starting with HOMEPAGE_VAR_ and HOMEPAGE_FILE_ are supported * The value of env var HOMEPAGE_VAR_XXX will replace {{HOMEPAGE_VAR_XXX}} in any config * The value of env var HOMEPAGE_FILE_XXX must be a file path, the contents of which will be used to replace {{HOMEPAGE_FILE_XXX}} in any config * If a substituted value contains a variable reference it may also be replaced, but the behavior is non-deterministic --- src/utils/config/api-response.js | 5 +++-- src/utils/config/config.js | 22 ++++++++++++++++++++-- src/utils/config/docker.js | 5 +++-- src/utils/config/kubernetes.js | 5 +++-- src/utils/config/service-helpers.js | 9 ++++++--- src/utils/config/widget-helpers.js | 5 +++-- 6 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/utils/config/api-response.js b/src/utils/config/api-response.js index 497ba6787..fad3022d6 100644 --- a/src/utils/config/api-response.js +++ b/src/utils/config/api-response.js @@ -4,7 +4,7 @@ import path from "path"; import yaml from "js-yaml"; -import checkAndCopyConfig, { getSettings } from "utils/config/config"; +import checkAndCopyConfig, { getSettings, substituteEnvironmentVars } from "utils/config/config"; import { servicesFromConfig, servicesFromDocker, @@ -28,7 +28,8 @@ export async function bookmarksResponse() { checkAndCopyConfig("bookmarks.yaml"); const bookmarksYaml = path.join(process.cwd(), "config", "bookmarks.yaml"); - const fileContents = await fs.readFile(bookmarksYaml, "utf8"); + const rawFileContents = await fs.readFile(bookmarksYaml, "utf8"); + const fileContents = substituteEnvironmentVars(rawFileContents); const bookmarks = yaml.load(fileContents); if (!bookmarks) return []; diff --git a/src/utils/config/config.js b/src/utils/config/config.js index 57a63f7b4..92f9d21d1 100644 --- a/src/utils/config/config.js +++ b/src/utils/config/config.js @@ -27,10 +27,28 @@ export default function checkAndCopyConfig(config) { } } +export function substituteEnvironmentVars(str) { + const homepageVarPrefix = "HOMEPAGE_VAR_"; + const homepageFilePrefix = "HOMEPAGE_FILE_"; + + let result = str; + Object.keys(process.env).forEach(key => { + if (key.startsWith(homepageVarPrefix)) { + result = result.replaceAll(`{{${key}}}`, process.env[key]); + } else if (key.startsWith(homepageFilePrefix)) { + const filename = process.env[key]; + const fileContents = readFileSync(filename, "utf8"); + result = result.replaceAll(`{{${key}}}`, fileContents); + } + }); + return result; +} + export function getSettings() { checkAndCopyConfig("settings.yaml"); const settingsYaml = join(process.cwd(), "config", "settings.yaml"); - const fileContents = readFileSync(settingsYaml, "utf8"); + const rawFileContents = readFileSync(settingsYaml, "utf8"); + const fileContents = substituteEnvironmentVars(rawFileContents); return yaml.load(fileContents) ?? {}; -} \ No newline at end of file +} diff --git a/src/utils/config/docker.js b/src/utils/config/docker.js index 7e1422e0f..288e8233e 100644 --- a/src/utils/config/docker.js +++ b/src/utils/config/docker.js @@ -3,13 +3,14 @@ import { readFileSync } from "fs"; import yaml from "js-yaml"; -import checkAndCopyConfig from "utils/config/config"; +import checkAndCopyConfig, { substituteEnvironmentVars } from "utils/config/config"; export default function getDockerArguments(server) { checkAndCopyConfig("docker.yaml"); const configFile = path.join(process.cwd(), "config", "docker.yaml"); - const configData = readFileSync(configFile, "utf8"); + const rawConfigData = readFileSync(configFile, "utf8"); + const configData = substituteEnvironmentVars(rawConfigData); const servers = yaml.load(configData); if (!server) { diff --git a/src/utils/config/kubernetes.js b/src/utils/config/kubernetes.js index d44b75f3a..ba2a37a04 100644 --- a/src/utils/config/kubernetes.js +++ b/src/utils/config/kubernetes.js @@ -4,13 +4,14 @@ import { readFileSync } from "fs"; import yaml from "js-yaml"; import { KubeConfig } from "@kubernetes/client-node"; -import checkAndCopyConfig from "utils/config/config"; +import checkAndCopyConfig, { substituteEnvironmentVars } from "utils/config/config"; export default function getKubeConfig() { checkAndCopyConfig("kubernetes.yaml"); const configFile = path.join(process.cwd(), "config", "kubernetes.yaml"); - const configData = readFileSync(configFile, "utf8"); + const rawConfigData = readFileSync(configFile, "utf8"); + const configData = substituteEnvironmentVars(rawConfigData); const config = yaml.load(configData); const kc = new KubeConfig(); diff --git a/src/utils/config/service-helpers.js b/src/utils/config/service-helpers.js index b22e8baed..49401ee63 100644 --- a/src/utils/config/service-helpers.js +++ b/src/utils/config/service-helpers.js @@ -7,17 +7,19 @@ import * as shvl from "shvl"; import { NetworkingV1Api } from "@kubernetes/client-node"; import createLogger from "utils/logger"; -import checkAndCopyConfig from "utils/config/config"; +import checkAndCopyConfig, { substituteEnvironmentVars } from "utils/config/config"; import getDockerArguments from "utils/config/docker"; import getKubeConfig from "utils/config/kubernetes"; const logger = createLogger("service-helpers"); + export async function servicesFromConfig() { checkAndCopyConfig("services.yaml"); const servicesYaml = path.join(process.cwd(), "config", "services.yaml"); - const fileContents = await fs.readFile(servicesYaml, "utf8"); + const rawFileContents = await fs.readFile(servicesYaml, "utf8"); + const fileContents = substituteEnvironmentVars(rawFileContents); const services = yaml.load(fileContents); if (!services) { @@ -49,7 +51,8 @@ export async function servicesFromDocker() { checkAndCopyConfig("docker.yaml"); const dockerYaml = path.join(process.cwd(), "config", "docker.yaml"); - const dockerFileContents = await fs.readFile(dockerYaml, "utf8"); + const rawDockerFileContents = await fs.readFile(dockerYaml, "utf8"); + const dockerFileContents = substituteEnvironmentVars(rawDockerFileContents); const servers = yaml.load(dockerFileContents); if (!servers) { diff --git a/src/utils/config/widget-helpers.js b/src/utils/config/widget-helpers.js index 1c5398c27..c03bd9065 100644 --- a/src/utils/config/widget-helpers.js +++ b/src/utils/config/widget-helpers.js @@ -3,7 +3,7 @@ import path from "path"; import yaml from "js-yaml"; -import checkAndCopyConfig from "utils/config/config"; +import checkAndCopyConfig, { substituteEnvironmentVars } from "utils/config/config"; const exemptWidgets = ["search"]; @@ -11,7 +11,8 @@ export async function widgetsFromConfig() { checkAndCopyConfig("widgets.yaml"); const widgetsYaml = path.join(process.cwd(), "config", "widgets.yaml"); - const fileContents = await fs.readFile(widgetsYaml, "utf8"); + const rawFileContents = await fs.readFile(widgetsYaml, "utf8"); + const fileContents = substituteEnvironmentVars(rawFileContents); const widgets = yaml.load(fileContents); if (!widgets) return []; From b2d22d757487cb3ce18740275fe6f6e6284e6298 Mon Sep 17 00:00:00 2001 From: shamoon <4887959+shamoon@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:51:28 -0800 Subject: [PATCH 2/2] Cache config env variables & check if config contains variables before substitution --- src/utils/config/config.js | 39 ++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/src/utils/config/config.js b/src/utils/config/config.js index 92f9d21d1..1644917ea 100644 --- a/src/utils/config/config.js +++ b/src/utils/config/config.js @@ -2,8 +2,13 @@ import { join } from "path"; import { existsSync, copyFile, readFileSync } from "fs"; +import cache from "memory-cache"; import yaml from "js-yaml"; +const cacheKey = "homepageEnvironmentVariables"; +const homepageVarPrefix = "HOMEPAGE_VAR_"; +const homepageFilePrefix = "HOMEPAGE_FILE_"; + export default function checkAndCopyConfig(config) { const configYaml = join(process.cwd(), "config", config); if (!existsSync(configYaml)) { @@ -27,20 +32,30 @@ export default function checkAndCopyConfig(config) { } } -export function substituteEnvironmentVars(str) { - const homepageVarPrefix = "HOMEPAGE_VAR_"; - const homepageFilePrefix = "HOMEPAGE_FILE_"; +function getCachedEnvironmentVars() { + let cachedVars = cache.get(cacheKey); + if (!cachedVars) { + // initialize cache + cachedVars = Object.entries(process.env).filter(([key, ]) => key.includes(homepageVarPrefix) || key.includes(homepageFilePrefix)); + cache.put(cacheKey, cachedVars); + } + return cachedVars; +} +export function substituteEnvironmentVars(str) { let result = str; - Object.keys(process.env).forEach(key => { - if (key.startsWith(homepageVarPrefix)) { - result = result.replaceAll(`{{${key}}}`, process.env[key]); - } else if (key.startsWith(homepageFilePrefix)) { - const filename = process.env[key]; - const fileContents = readFileSync(filename, "utf8"); - result = result.replaceAll(`{{${key}}}`, fileContents); - } - }); + if (result.includes('{{')) { // crude check if we have vars to replace + const cachedVars = getCachedEnvironmentVars(); + cachedVars.forEach(([key, value]) => { + if (key.startsWith(homepageVarPrefix)) { + result = result.replaceAll(`{{${key}}}`, value); + } else if (key.startsWith(homepageFilePrefix)) { + const filename = value; + const fileContents = readFileSync(filename, "utf8"); + result = result.replaceAll(`{{${key}}}`, fileContents); + } + }); + } return result; }