From 12370e435abf9a38e2eb689975d5b9e238df4ae5 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:01:52 -0400 Subject: [PATCH] feat(compose): list the variables a compose file uses in its Environment tab The Environment tab of a compose service only showed what was set in the panel. Every `${NAME:-default}` in the compose file stayed out of sight until someone opened the file, so it was hard to tell which settings a service has and which values it falls back to. A "Variables from compose" button in the Environment tab now fetches the source selected in General (the same clone that Preview Compose and deploys use, for every git provider and for raw files), reads the compose file and lists the variables it uses: the default the file falls back to, the services that use it, and whether the environment already sets it. Names that look like credentials, defaults that ask to be replaced (`change-me`), required variables (`${NAME:?}`) and variables with different defaults in different places are marked. The selected variables go into the editor. Nothing is saved until Save is clicked, and a variable that is already set is never changed. By default they are added commented out (`# NAME=value`): dotenv skips those lines, so the defaults from the file keep applying and later changes to them in the repository still take effect, while removing the `#` turns a line into an override. Adding again rewrites the commented lines in place, so they follow the current defaults. A switch adds active lines instead, for a fixed copy of the values. Details: - Variables are read from the parsed YAML, so references inside YAML comments are not listed, and anchors count for the services that use them. `$$` is a literal dollar sign, and references inside a default (`${A:-${B}}`) are listed too. APP_NAME, COMPOSE_PROJECT_NAME, COMPOSE_PREFIX and DOCKER_CONFIG are left out because Dokploy writes them itself. - Values are quoted where dotenv would otherwise cut or change them (`#`, spaces, quotes, backslashes). - compose.loadComposeVariables needs service create (it fetches the source, like fetchSourceType) and env read (defaults are often secrets). It refuses while a deployment is running, because fetching replaces the code directory the build reads from. API keys cannot call it, like revealEnvironment. - Opening the dialog reveals a masked environment first, because a masked editor with unsaved changes cannot be revealed afterwards. - When "Create Environment File" is off, the dialog says that the variables do not reach the compose file. Co-Authored-By: Claude Opus 5.5 --- .../compose/compose-env-merge.test.ts | 167 ++++++++ .../compose/compose-variables.test.ts | 225 +++++++++++ .../security/api-key-output-guard.test.ts | 5 + .../service-environment-reveal.test.ts | 22 ++ .../environment/show-environment.tsx | 53 ++- .../environment/compose-variables-dialog.tsx | 371 ++++++++++++++++++ apps/dokploy/lib/compose-env.ts | 111 ++++++ apps/dokploy/server/api/routers/compose.ts | 13 + .../server/api/utils/api-key-output-guard.ts | 1 + .../server/api/utils/procedure-access.ts | 1 + packages/server/src/index.ts | 1 + packages/server/src/services/compose.ts | 72 ++++ .../src/utils/docker/compose-variables.ts | 181 +++++++++ 13 files changed, 1222 insertions(+), 1 deletion(-) create mode 100644 apps/dokploy/__test__/compose/compose-env-merge.test.ts create mode 100644 apps/dokploy/__test__/compose/compose-variables.test.ts create mode 100644 apps/dokploy/components/dashboard/compose/environment/compose-variables-dialog.tsx create mode 100644 apps/dokploy/lib/compose-env.ts create mode 100644 packages/server/src/utils/docker/compose-variables.ts diff --git a/apps/dokploy/__test__/compose/compose-env-merge.test.ts b/apps/dokploy/__test__/compose/compose-env-merge.test.ts new file mode 100644 index 00000000000..e58139a2996 --- /dev/null +++ b/apps/dokploy/__test__/compose/compose-env-merge.test.ts @@ -0,0 +1,167 @@ +import { parse } from "dotenv"; +import { describe, expect, it } from "vitest"; +import { + formatEnvValue, + mergeComposeEnv, + readComposeEnvState, +} from "@/lib/compose-env"; + +const HEADER = ["Defaults from ./docker-compose.yml (branch main)."]; + +describe("formatEnvValue", () => { + it.each([ + ["8080", "8080"], + ["", ""], + ["http://origin:80", "http://origin:80"], + ["a b", "'a b'"], + ["color #fff", "'color #fff'"], + ["it's", `"it's"`], + [`it's "quoted"`, `\`it's "quoted"\``], + ["C:\\path's", "`C:\\path's`"], + ])("writes %j so dotenv reads it back unchanged", (value, formatted) => { + expect(formatEnvValue(value)).toBe(formatted); + expect(parse(`KEY=${formatted}`).KEY).toBe(value); + }); +}); + +describe("readComposeEnvState", () => { + it("tells set, listed and absent variables apart", () => { + const stateOf = readComposeEnvState( + [ + "SET=1", + "export EXPORTED=2", + "# LISTED=3", + "# BOTH=old", + "BOTH=new", + "# a note about PROSE = text", + ].join("\n"), + ); + + expect(stateOf("SET")).toEqual({ kind: "set" }); + expect(stateOf("EXPORTED")).toEqual({ kind: "set" }); + expect(stateOf("LISTED")).toEqual({ kind: "listed", value: "3" }); + expect(stateOf("BOTH")).toEqual({ kind: "set" }); + expect(stateOf("PROSE")).toEqual({ kind: "absent" }); + expect(stateOf("MISSING")).toEqual({ kind: "absent" }); + }); +}); + +describe("mergeComposeEnv", () => { + it("adds commented defaults that dotenv does not apply", () => { + const merged = mergeComposeEnv( + "", + [ + { name: "PORT", value: "8080", services: ["web"] }, + { name: "MODE", value: "off", services: ["web"] }, + ], + { active: false, header: HEADER }, + ); + + expect(merged).toBe( + [ + "# Defaults from ./docker-compose.yml (branch main).", + "# PORT=8080", + "# MODE=off", + "", + ].join("\n"), + ); + expect(parse(merged)).toEqual({}); + }); + + it("adds active lines that dotenv applies", () => { + const merged = mergeComposeEnv( + "", + [{ name: "MODE", value: "a b", services: ["web"] }], + { active: true, header: HEADER }, + ); + + expect(parse(merged)).toEqual({ MODE: "a b" }); + }); + + it("never changes a variable that is already set", () => { + const merged = mergeComposeEnv( + "PORT=9000\nSECRET=mine", + [ + { name: "PORT", value: "8080", services: ["web"] }, + { name: "SECRET", value: "change-me", services: ["web"] }, + ], + { active: true, header: HEADER }, + ); + + expect(merged).toBe("PORT=9000\nSECRET=mine"); + }); + + it("refreshes commented defaults in place instead of adding them again", () => { + const env = "KEEP=1\n# LIMIT=60\n# my own note\nOTHER=2\n"; + + expect( + mergeComposeEnv( + env, + [{ name: "LIMIT", value: "100", services: ["engine"] }], + { active: false, header: HEADER }, + ), + ).toBe("KEEP=1\n# LIMIT=100\n# my own note\nOTHER=2\n"); + + expect( + mergeComposeEnv( + env, + [{ name: "LIMIT", value: "100", services: ["engine"] }], + { active: true, header: HEADER }, + ), + ).toBe("KEEP=1\nLIMIT=100\n# my own note\nOTHER=2\n"); + }); + + it("appends after the existing text and groups lines by service", () => { + const merged = mergeComposeEnv( + "KEEP=1\n\n", + [ + { name: "HTTP_PORT", value: "8080", services: ["caddy"] }, + { + name: "REDIS_URL", + value: "redis://redis:6379", + services: ["engine"], + }, + { name: "PATHS", value: "/hook", services: ["caddy", "engine"] }, + { name: "TLS", value: "off", services: ["caddy"] }, + { name: "NETWORK", value: "edge", services: [] }, + ], + { + active: false, + header: [...HEADER, "Remove the # to override a default."], + }, + ); + + expect(merged).toBe( + [ + "KEEP=1", + "", + "# Defaults from ./docker-compose.yml (branch main).", + "# Remove the # to override a default.", + "", + "# Used by caddy", + "# HTTP_PORT=8080", + "# TLS=off", + "", + "# Used by engine", + "# REDIS_URL=redis://redis:6379", + "", + "# Used by caddy, engine", + "# PATHS=/hook", + "", + "# Not tied to a service", + "# NETWORK=edge", + "", + ].join("\n"), + ); + expect(parse(merged)).toEqual({ KEEP: "1" }); + }); + + it("leaves the text alone when nothing is left to add", () => { + expect( + mergeComposeEnv("A=1", [{ name: "A", value: "2", services: [] }], { + active: false, + header: HEADER, + }), + ).toBe("A=1"); + }); +}); diff --git a/apps/dokploy/__test__/compose/compose-variables.test.ts b/apps/dokploy/__test__/compose/compose-variables.test.ts new file mode 100644 index 00000000000..756a5cb5088 --- /dev/null +++ b/apps/dokploy/__test__/compose/compose-variables.test.ts @@ -0,0 +1,225 @@ +import { + extractComposeVariables, + findVariableReferences, +} from "@dokploy/server/utils/docker/compose-variables"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const variablesOf = (yaml: string) => + extractComposeVariables(parse(yaml, { maxAliasCount: 10000 })); + +describe("findVariableReferences", () => { + it("reads every interpolation form compose supports", () => { + expect( + findVariableReferences( + "${A} ${B:-one} ${C-two} ${D:?need D} ${E?need E} ${F:+on} $G", + ), + ).toEqual([ + { name: "A", operator: null, value: null }, + { name: "B", operator: ":-", value: "one" }, + { name: "C", operator: "-", value: "two" }, + { name: "D", operator: ":?", value: "need D" }, + { name: "E", operator: "?", value: "need E" }, + { name: "F", operator: ":+", value: "on" }, + { name: "G", operator: null, value: null }, + ]); + }); + + it("treats $$ as a literal dollar sign", () => { + expect(findVariableReferences("echo $$HOME $${NOT_A_VAR} ${REAL}")).toEqual( + [{ name: "REAL", operator: null, value: null }], + ); + }); + + it("keeps nested defaults whole and reads the variables inside them", () => { + expect( + findVariableReferences("${URL:-http://${HOST:-localhost}:80}"), + ).toEqual([ + { + name: "URL", + operator: ":-", + value: "http://${HOST:-localhost}:80", + }, + { name: "HOST", operator: ":-", value: "localhost" }, + ]); + }); + + it("ignores text that only looks like a reference", () => { + expect(findVariableReferences("$ ${ ${1A} ${UNCLOSED:-x ${A B}")).toEqual( + [], + ); + }); +}); + +describe("extractComposeVariables", () => { + it("lists variables with their defaults and the services using them", () => { + const variables = variablesOf(` +services: + caddy: + image: caddy + ports: + - "\${HTTP_PORT:-8080}:80" + environment: + MODE: \${MODE:-off} + PATHS: \${PATHS:-/payments/webhook} + engine: + image: engine + environment: + PATHS: \${PATHS:-/payments/webhook} + TOKEN: \${ADMIN_TOKEN:-} + HOST: \${DB_HOST} + REQUIRED: \${DB_PASSWORD:?set a password} +`); + + expect(variables).toEqual([ + { + name: "HTTP_PORT", + defaultValue: "8080", + required: false, + conflictingDefaults: false, + services: ["caddy"], + secret: false, + placeholder: false, + }, + { + name: "MODE", + defaultValue: "off", + required: false, + conflictingDefaults: false, + services: ["caddy"], + secret: false, + placeholder: false, + }, + { + name: "PATHS", + defaultValue: "/payments/webhook", + required: false, + conflictingDefaults: false, + services: ["caddy", "engine"], + secret: false, + placeholder: false, + }, + { + name: "ADMIN_TOKEN", + defaultValue: "", + required: false, + conflictingDefaults: false, + services: ["engine"], + secret: true, + placeholder: false, + }, + { + name: "DB_HOST", + defaultValue: null, + required: false, + conflictingDefaults: false, + services: ["engine"], + secret: false, + placeholder: false, + }, + { + name: "DB_PASSWORD", + defaultValue: null, + required: true, + conflictingDefaults: false, + services: ["engine"], + secret: true, + placeholder: false, + }, + ]); + }); + + it("flags defaults that ask to be replaced", () => { + const variables = variablesOf(` +services: + engine: + environment: + SALT: \${HASH_SALT:-change-me-hash-salt} + ADMIN: \${CAP_ADMIN_KEY:-replace_me} + TTL: \${PASS_TTL_SECONDS:-1800} +`); + + expect( + variables.map(({ name, secret, placeholder }) => [ + name, + secret, + placeholder, + ]), + ).toEqual([ + ["HASH_SALT", false, true], + ["CAP_ADMIN_KEY", true, true], + ["PASS_TTL_SECONDS", true, false], + ]); + }); + + it("skips variables that appear only in YAML comments", () => { + const variables = variablesOf(` +services: + cap: + # The dashboard listens on 127.0.0.1:\${DASHBOARD_PORT} of the host. + ports: + - "127.0.0.1:\${DASHBOARD_PORT:-3100}:3000" + # \${ONLY_IN_A_COMMENT:-x} +`); + + expect( + variables.map(({ name, defaultValue }) => [name, defaultValue]), + ).toEqual([["DASHBOARD_PORT", "3100"]]); + }); + + it("follows anchors into the services that use them", () => { + const variables = variablesOf(` +x-common: &common + environment: + LOG_LEVEL: \${LOG_LEVEL:-info} +services: + api: + <<: *common + image: api + worker: + <<: *common + image: worker +`); + + expect(variables).toMatchObject([ + { name: "LOG_LEVEL", defaultValue: "info", services: ["api", "worker"] }, + ]); + }); + + it("flags a variable whose defaults differ between places", () => { + const variables = variablesOf(` +services: + a: + environment: + LIMIT: \${LIMIT:-60} + b: + environment: + LIMIT: \${LIMIT:-120} +`); + + expect(variables).toMatchObject([ + { name: "LIMIT", defaultValue: "60", conflictingDefaults: true }, + ]); + }); + + it("keeps variables outside services and leaves out the ones Dokploy sets", () => { + const variables = variablesOf(` +services: + web: + container_name: \${APP_NAME}-web + image: \${COMPOSE_PROJECT_NAME}:latest +networks: + edge: + name: \${EDGE_NETWORK:-edge} +`); + + expect(variables).toMatchObject([ + { name: "EDGE_NETWORK", defaultValue: "edge", services: [] }, + ]); + }); + + it("returns nothing for an empty or non-mapping file", () => { + expect(extractComposeVariables(null)).toEqual([]); + expect(extractComposeVariables(["${A}"])).toEqual([]); + }); +}); diff --git a/apps/dokploy/__test__/security/api-key-output-guard.test.ts b/apps/dokploy/__test__/security/api-key-output-guard.test.ts index 68a2381cfa0..ffab871c21f 100644 --- a/apps/dokploy/__test__/security/api-key-output-guard.test.ts +++ b/apps/dokploy/__test__/security/api-key-output-guard.test.ts @@ -69,6 +69,7 @@ const { restoreApiKeyText } = await import( const testRouter = createTRPCRouter({ compose: createTRPCRouter({ + loadComposeVariables: secretReader, one: protectedProcedure.query(() => ({ composeId: "compose-1", composeFile: storedCompose, @@ -263,6 +264,10 @@ describe("procedures that return raw secrets", () => { () => apiKey().application.revealEnvironment(), ], ["postgres.revealEnvironment", () => apiKey().postgres.revealEnvironment()], + [ + "compose.loadComposeVariables", + () => apiKey().compose.loadComposeVariables(), + ], ["docker.getConfig", () => apiKey().docker.getConfig()], ["docker.readContainerFile", () => apiKey().docker.readContainerFile()], [ diff --git a/apps/dokploy/__test__/security/service-environment-reveal.test.ts b/apps/dokploy/__test__/security/service-environment-reveal.test.ts index 19292261dbe..4cf5efd7b73 100644 --- a/apps/dokploy/__test__/security/service-environment-reveal.test.ts +++ b/apps/dokploy/__test__/security/service-environment-reveal.test.ts @@ -36,6 +36,7 @@ const serverMocks = vi.hoisted(() => ({ getContainerLogs: vi.fn(), getContainersByAppNameMatch: vi.fn(), getWebServerSettings: vi.fn(), + loadComposeVariables: vi.fn(), loadServices: vi.fn(), mechanizeDockerContainer: vi.fn(), randomizeComposeFile: vi.fn(), @@ -114,6 +115,7 @@ vi.mock("@dokploy/server", () => ({ getContainerLogs: serverMocks.getContainerLogs, getContainersByAppNameMatch: serverMocks.getContainersByAppNameMatch, getWebServerSettings: serverMocks.getWebServerSettings, + loadComposeVariables: serverMocks.loadComposeVariables, loadServices: serverMocks.loadServices, mechanizeDockerContainer: serverMocks.mechanizeDockerContainer, randomizeComposeFile: serverMocks.randomizeComposeFile, @@ -337,6 +339,26 @@ describe("service environment reveal boundary", () => { }); }); + it("reads compose variables only for callers that can read env vars", async () => { + permissionMocks.checkServicePermissionAndAccess.mockRejectedValueOnce( + new TRPCError({ code: "UNAUTHORIZED", message: "denied" }), + ); + + await expect( + composeRouter + .createCaller(createContext()) + .loadComposeVariables({ composeId: "compose-1" }), + ).rejects.toMatchObject({ code: "UNAUTHORIZED" }); + + expect( + permissionMocks.checkServicePermissionAndAccess, + ).toHaveBeenCalledWith(expect.anything(), "compose-1", { + service: ["create"], + envVars: ["read"], + }); + expect(serverMocks.loadComposeVariables).not.toHaveBeenCalled(); + }); + it("keeps the compose file for callers that can read env vars", async () => { permissionMocks.hasPermission.mockResolvedValueOnce(true); diff --git a/apps/dokploy/components/dashboard/application/environment/show-environment.tsx b/apps/dokploy/components/dashboard/application/environment/show-environment.tsx index cd0bd414d0f..c59b5239208 100644 --- a/apps/dokploy/components/dashboard/application/environment/show-environment.tsx +++ b/apps/dokploy/components/dashboard/application/environment/show-environment.tsx @@ -4,6 +4,10 @@ import { type CSSProperties, useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import { z } from "zod"; +import { + type ComposeEnvImport, + ComposeVariablesDialog, +} from "@/components/dashboard/compose/environment/compose-variables-dialog"; import { CodeEditor } from "@/components/shared/code-editor"; import { useEnvCompletionSource } from "@/components/shared/env-autocomplete"; import { VaultImportDialog } from "@/components/shared/vault-import-dialog"; @@ -26,6 +30,7 @@ import { } from "@/components/ui/form"; import { Switch } from "@/components/ui/switch"; import { Toggle } from "@/components/ui/toggle"; +import { mergeComposeEnv } from "@/lib/compose-env"; import { api } from "@/utils/api"; import type { ServiceType } from "../advanced/show-resources"; @@ -44,6 +49,11 @@ interface Props { export const ShowEnvironment = ({ id, type }: Props) => { const { data: permissions } = api.user.getPermissions.useQuery(); const canWrite = permissions?.envVars.write ?? false; + const canImportComposeVariables = + type === "compose" && + canWrite && + (permissions?.envVars.read ?? false) && + (permissions?.service.create ?? false); const queryMap = { compose: () => api.compose.one.useQuery({ composeId: id }, { enabled: !!id }), @@ -192,6 +202,38 @@ export const ShowEnvironment = ({ id, type }: Props) => { } }; + // Imported lines go into the revealed text: a masked editor cannot be + // revealed again while it holds unsaved changes. + const revealForEditing = async () => { + if (isEnvVisible) { + try { + await handleReveal(); + } catch { + return false; + } + setIsEnvVisible(false); + } + return true; + }; + + const handleComposeImport = ({ + entries, + active, + header, + }: ComposeEnvImport) => { + form.setValue( + "environment", + mergeComposeEnv(form.getValues("environment"), entries, { + active, + header, + }), + { shouldDirty: true }, + ); + toast.success( + `Added ${entries.length} variable${entries.length === 1 ? "" : "s"}. Review them and click Save.`, + ); + }; + // Add keyboard shortcut for Ctrl+S/Cmd+S useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { @@ -257,7 +299,16 @@ export const ShowEnvironment = ({ id, type }: Props) => { name="environment" render={({ field }) => ( -
+
+ {canImportComposeVariables && ( + + )} Promise; + onImport: (changes: ComposeEnvImport) => void; +} + +const compareWithEnv = ( + stateOf: ReturnType, + name: string, + value: string, +) => { + const state = stateOf(name); + const listedValue = state.kind === "listed" ? state.value.trim() : null; + return { + state: state.kind, + listedValue, + changed: listedValue !== null && listedValue !== formatEnvValue(value), + }; +}; + +export const ComposeVariablesDialog = ({ + composeId, + currentEnv, + createEnvFile, + beforeOpen, + onImport, +}: Props) => { + const [isOpen, setIsOpen] = useState(false); + const [isPreparing, setIsPreparing] = useState(false); + const [filter, setFilter] = useState(""); + const [selected, setSelected] = useState>(new Set()); + const [active, setActive] = useState(false); + const { mutateAsync, data, error, isPending, reset } = + api.compose.loadComposeVariables.useMutation(); + // beforeOpen may reveal the environment, which arrives after load() began. + const currentEnvRef = useRef(currentEnv); + currentEnvRef.current = currentEnv; + + const rows = useMemo(() => { + const stateOf = readComposeEnvState(currentEnv); + return (data?.variables ?? []).map((variable) => { + const value = variable.defaultValue ?? ""; + return { + ...variable, + value, + ...compareWithEnv(stateOf, variable.name, value), + }; + }); + }, [data, currentEnv]); + + const visibleRows = useMemo(() => { + const query = filter.trim().toLowerCase(); + if (!query) { + return rows; + } + return rows.filter( + (row) => + row.name.toLowerCase().includes(query) || + row.services.some((service) => service.toLowerCase().includes(query)), + ); + }, [rows, filter]); + + const selectableRows = visibleRows.filter((row) => row.state !== "set"); + const setCount = rows.filter((row) => row.state === "set").length; + + const load = async () => { + setSelected(new Set()); + try { + const result = await mutateAsync({ composeId }); + const stateOf = readComposeEnvState(currentEnvRef.current); + setSelected( + new Set( + result.variables + .filter((variable) => { + const { state, changed } = compareWithEnv( + stateOf, + variable.name, + variable.defaultValue ?? "", + ); + return state === "absent" || changed; + }) + .map((variable) => variable.name), + ), + ); + } catch { + // The error is shown in the dialog. + } + }; + + const handleOpenChange = async (open: boolean) => { + if (!open) { + setIsOpen(false); + setFilter(""); + reset(); + return; + } + setIsPreparing(true); + try { + if (!(await beforeOpen())) { + return; + } + } finally { + setIsPreparing(false); + } + setIsOpen(true); + await load(); + }; + + const toggleRow = (name: string, checked: boolean) => { + setSelected((previous) => { + const next = new Set(previous); + if (checked) { + next.add(name); + } else { + next.delete(name); + } + return next; + }); + }; + + const allVisibleSelected = + selectableRows.length > 0 && + selectableRows.every((row) => selected.has(row.name)); + + const toggleAllVisible = () => { + setSelected((previous) => { + const next = new Set(previous); + for (const row of selectableRows) { + if (allVisibleSelected) { + next.delete(row.name); + } else { + next.add(row.name); + } + } + return next; + }); + }; + + const handleImport = () => { + if (!data) { + return; + } + const source = `${data.composePath}${data.branch ? ` (branch ${data.branch})` : ""}`; + onImport({ + entries: rows + .filter((row) => row.state !== "set" && selected.has(row.name)) + .map(({ name, value, services }) => ({ name, value, services })), + active, + header: active + ? [`Values copied from ${source}.`] + : [ + `Defaults from ${source}.`, + "Lines starting with # are not applied: remove the # and change the value to override a default.", + ], + }); + handleOpenChange(false); + }; + + const selectedCount = rows.filter( + (row) => row.state !== "set" && selected.has(row.name), + ).length; + + return ( + + + + + Variables from the compose file + + {data + ? `Variables used in ${data.composePath}${data.branch ? ` on branch ${data.branch}` : ""}, with the value the file falls back to while a variable is not set here.` + : "Fetching the compose file from the source selected in General."} + + + +
+ {!createEnvFile && ( + + Create Environment File is off, so variables set here do not reach + the compose file until it is turned on. + + )} + + {isPending || (!data && !error) ? ( +
+ + Fetching the compose file... +
+ ) : error ? ( +
+ {error.message} + +
+ ) : rows.length === 0 ? ( +
+ The compose file does not use any variables. +
+ ) : ( + <> +
+ {rows.length} variables: {setCount} set here,{" "} + {rows.length - setCount} use the default from the file. +
+ setFilter(event.target.value)} + /> +
+ + + {selectedCount} selected + +
+ +
+ {visibleRows.map((row) => ( +
+ + toggleRow(row.name, checked === true) + } + /> +
+
+ + {row.state === "set" && ( + set here + )} + {row.state === "listed" && !row.changed && ( + listed + )} + {row.changed && ( + default changed + )} + {row.secret && secret} + {row.placeholder && ( + placeholder + )} + {row.required && ( + required + )} + {row.conflictingDefaults && ( + defaults differ + )} +
+ + {row.defaultValue === null + ? "no default" + : row.defaultValue === "" + ? "empty by default" + : `default: ${row.defaultValue}`} + + {row.changed && ( + + listed here as: {row.listedValue} + + )} + {row.services.length > 0 && ( + + Used by {row.services.join(", ")} + + )} + {row.placeholder && ( + + The default from the file is a placeholder. Set your + own value. + + )} +
+
+ ))} +
+ + + )} + +
+
+ +

+ {active + ? "Values are copied as they are and override the file, so later changes to the defaults in the repository no longer apply." + : "Lines are added commented out (# NAME=value): the defaults from the file keep applying, and you override one by removing its #. Adding again refreshes the commented values."} +

+
+ +
+
+ + + + + + + ); +}; diff --git a/apps/dokploy/lib/compose-env.ts b/apps/dokploy/lib/compose-env.ts new file mode 100644 index 00000000000..c673d4110c1 --- /dev/null +++ b/apps/dokploy/lib/compose-env.ts @@ -0,0 +1,111 @@ +export interface ComposeEnvEntry { + name: string; + value: string; + services: string[]; +} + +export type ComposeEnvState = + | { kind: "set" } + | { kind: "listed"; value: string } + | { kind: "absent" }; + +const ACTIVE_LINE = /^\s*(?:export\s+)?([A-Za-z_][\w.-]*)\s*=/; +// Only `# NAME=value` with nothing between the name and "=", so prose in +// comments is not taken for a variable. +const COMMENTED_LINE = /^\s*#\s*([A-Za-z_]\w*)=(.*)$/; + +// dotenv ends an unquoted value at " #" and trims it, and expands escapes such +// as \n only inside double quotes. +export const formatEnvValue = (value: string) => { + if (!/[\s#"'`\\]/.test(value)) { + return value; + } + if (!value.includes("'")) { + return `'${value}'`; + } + if (!value.includes('"') && !value.includes("\\")) { + return `"${value}"`; + } + return `\`${value}\``; +}; + +const findCommentedLines = (lines: string[]) => { + const commented = new Map(); + lines.forEach((line, index) => { + const name = COMMENTED_LINE.exec(line)?.[1]; + if (name && !commented.has(name)) { + commented.set(name, index); + } + }); + return commented; +}; + +export const readComposeEnvState = (env: string) => { + const lines = env.split("\n"); + const active = new Set(); + for (const line of lines) { + const name = ACTIVE_LINE.exec(line)?.[1]; + if (name) { + active.add(name); + } + } + const commented = findCommentedLines(lines); + return (name: string): ComposeEnvState => { + if (active.has(name)) { + return { kind: "set" }; + } + const index = commented.get(name); + if (index !== undefined) { + return { + kind: "listed", + value: COMMENTED_LINE.exec(lines[index] ?? "")?.[2] ?? "", + }; + } + return { kind: "absent" }; + }; +}; + +// Variables already set stay as they are. A commented `# NAME=value` line is +// rewritten in place, so importing again refreshes the defaults it shows; +// the rest go to a block at the end, grouped by the services that use them. +export const mergeComposeEnv = ( + env: string, + entries: ComposeEnvEntry[], + { active, header }: { active: boolean; header: string[] }, +) => { + const lines = env.split("\n"); + const state = readComposeEnvState(env); + const commented = findCommentedLines(lines); + const groups = new Map(); + + for (const entry of entries) { + if (state(entry.name).kind === "set") { + continue; + } + const assignment = `${entry.name}=${formatEnvValue(entry.value)}`; + const line = active ? assignment : `# ${assignment}`; + const index = commented.get(entry.name); + if (index !== undefined) { + lines[index] = line; + continue; + } + const group = entry.services.join(", "); + groups.set(group, [...(groups.get(group) ?? []), line]); + } + + const merged = lines.join("\n"); + if (groups.size === 0) { + return merged; + } + + const block = header.map((line) => `# ${line}`); + for (const [group, groupLines] of groups) { + if (groups.size > 1) { + block.push("", group ? `# Used by ${group}` : "# Not tied to a service"); + } + block.push(...groupLines); + } + + const base = merged.trimEnd(); + return `${base}${base ? "\n\n" : ""}${block.join("\n")}\n`; +}; diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index ce93c44a712..b4f8b31d664 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -23,6 +23,7 @@ import { getContainersByAppNameMatch, getWebServerSettings, IS_CLOUD, + loadComposeVariables, loadServices, markDeploymentOperationDispatched, randomizeComposeFile, @@ -549,6 +550,18 @@ export const composeRouter = createTRPCRouter({ }); return await loadServices(input.composeId, input.type); }), + loadComposeVariables: protectedProcedure + .input(apiFindCompose) + .mutation(async ({ input, ctx }) => { + // Defaults in a compose file are often secrets, so this needs the + // same access as reading the environment; fetching the source also + // needs what fetchSourceType needs. + await checkServicePermissionAndAccess(ctx, input.composeId, { + service: ["create"], + envVars: ["read"], + }); + return await loadComposeVariables(input.composeId); + }), loadMountsByService: protectedProcedure .input( z.object({ diff --git a/apps/dokploy/server/api/utils/api-key-output-guard.ts b/apps/dokploy/server/api/utils/api-key-output-guard.ts index 2fe60e7104e..5fccb147213 100644 --- a/apps/dokploy/server/api/utils/api-key-output-guard.ts +++ b/apps/dokploy/server/api/utils/api-key-output-guard.ts @@ -24,6 +24,7 @@ import { isApiKeySession } from "@/server/api/utils/super-session"; // make safe (a container file may be nothing but one token), so an API key // cannot call them at all. const API_KEY_DENIED_PROCEDURES = new Set([ + "compose.loadComposeVariables", "docker.getConfig", "docker.readContainerFile", "dockerVolume.readVolumeFile", diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index f2debf65233..3aeb3ec45a0 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -178,6 +178,7 @@ export const PROCEDURE_ACCESS: Record = { "compose.import": "write", "compose.isolatedDeployment": "write", "compose.killBuild": "write", + "compose.loadComposeVariables": "write", "compose.loadMountsByService": "read", "compose.loadServices": "read", "compose.move": "write", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 5f4d5b45f6f..9b3119c7ec4 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -118,6 +118,7 @@ export * from "./utils/docker/compose/network"; export * from "./utils/docker/compose/secrets"; export * from "./utils/docker/compose/service"; export * from "./utils/docker/compose/volume"; +export * from "./utils/docker/compose-variables"; export * from "./utils/docker/domain"; export * from "./utils/docker/types"; export * from "./utils/docker/utils"; diff --git a/packages/server/src/services/compose.ts b/packages/server/src/services/compose.ts index e03288ef41c..608b98b8788 100644 --- a/packages/server/src/services/compose.ts +++ b/packages/server/src/services/compose.ts @@ -10,6 +10,7 @@ import { } from "@dokploy/server/db/schema"; import { getBuildComposeCommand } from "@dokploy/server/utils/builders/compose"; import { randomizeSpecificationFile } from "@dokploy/server/utils/docker/compose"; +import { extractComposeVariables } from "@dokploy/server/utils/docker/compose-variables"; import { cloneCompose, loadDockerCompose, @@ -235,6 +236,77 @@ export const loadServices = async ( return [...services]; }; +const getComposeBranch = (compose: Compose) => { + switch (compose.sourceType) { + case "github": + return compose.branch; + case "gitlab": + return compose.gitlabBranch; + case "bitbucket": + return compose.bitbucketBranch; + case "gitea": + return compose.giteaBranch; + case "git": + return compose.customGitBranch; + default: + return null; + } +}; + +export const loadComposeVariables = async (composeId: string) => { + const compose = await findComposeById(composeId); + + // Fetching replaces the code directory that a running build reads from. + if (compose.composeStatus === "running") { + throw new TRPCError({ + code: "CONFLICT", + message: "A deployment is running. Try again when it finishes.", + }); + } + + try { + const command = await cloneCompose(compose); + if (compose.serverId) { + await execAsyncRemote(compose.serverId, command); + } else { + await execAsync(command); + } + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Could not fetch the source of this compose service", + cause: error, + }); + } + + let composeData: ComposeSpecification | null; + try { + composeData = compose.serverId + ? await loadDockerComposeRemote(compose) + : await loadDockerCompose(compose); + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "The compose file is not valid YAML", + cause: error, + }); + } + + if (!composeData) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Compose file not found", + }); + } + + return { + composePath: + compose.sourceType === "raw" ? "docker-compose.yml" : compose.composePath, + branch: getComposeBranch(compose), + variables: extractComposeVariables(composeData), + }; +}; + export const updateCompose = async ( composeId: string, composeData: Partial, diff --git a/packages/server/src/utils/docker/compose-variables.ts b/packages/server/src/utils/docker/compose-variables.ts new file mode 100644 index 00000000000..82d1c46831f --- /dev/null +++ b/packages/server/src/utils/docker/compose-variables.ts @@ -0,0 +1,181 @@ +import { isSecretEnvName } from "../env-upsert"; + +export interface ComposeVariableReference { + name: string; + operator: string | null; + value: string | null; +} + +export interface ComposeVariable { + name: string; + // null when the file gives no default, so the variable is empty unless set. + defaultValue: string | null; + // `${NAME:?message}`: compose refuses to start while the variable is unset. + required: boolean; + // Different places in the file fall back to different values. + conflictingDefaults: boolean; + services: string[]; + secret: boolean; + // The default asks to be replaced, such as `change-me-salt`. + placeholder: boolean; +} + +// Dokploy writes these into the .env file of every compose service itself. +const DOKPLOY_PROVIDED_VARIABLES = new Set([ + "APP_NAME", + "COMPOSE_PROJECT_NAME", + "COMPOSE_PREFIX", + "DOCKER_CONFIG", +]); + +const PLACEHOLDER_PATTERN = /change[-_]?me|replace[-_]?me/i; + +const NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*/; +const OPERATOR_PATTERN = /^:?[-?+]/; + +const findClosingBrace = (text: string, start: number) => { + let depth = 1; + let index = start; + while (index < text.length) { + if (text[index] === "$" && text[index + 1] === "$") { + index += 2; + continue; + } + if (text[index] === "$" && text[index + 1] === "{") { + depth++; + index += 2; + continue; + } + if (text[index] === "}" && --depth === 0) { + return index; + } + index++; + } + return -1; +}; + +// Follows compose interpolation: `$$` is a literal dollar sign, and a value +// after an operator may hold further references (`${A:-${B:-x}}`). +export const findVariableReferences = ( + text: string, +): ComposeVariableReference[] => { + const references: ComposeVariableReference[] = []; + let index = text.indexOf("$"); + while (index !== -1) { + const next = text[index + 1]; + let resumeAt = index + 1; + if (next === "$") { + resumeAt = index + 2; + } else if (next === "{") { + const name = NAME_PATTERN.exec(text.slice(index + 2))?.[0]; + const afterName = index + 2 + (name?.length ?? 0); + const operator = + OPERATOR_PATTERN.exec(text.slice(afterName))?.[0] ?? null; + const valueStart = afterName + (operator?.length ?? 0); + const end = name ? findClosingBrace(text, valueStart) : -1; + if (name && end !== -1 && (operator || end === valueStart)) { + const value = operator ? text.slice(valueStart, end) : null; + references.push({ name, operator, value }); + if (value) { + references.push(...findVariableReferences(value)); + } + resumeAt = end + 1; + } + } else { + const name = NAME_PATTERN.exec(text.slice(index + 1))?.[0]; + if (name) { + references.push({ name, operator: null, value: null }); + resumeAt = index + 1 + name.length; + } + } + index = text.indexOf("$", resumeAt); + } + return references; +}; + +const collectStrings = ( + value: unknown, + strings: string[], + ancestors = new Set(), +) => { + if (typeof value === "string") { + strings.push(value); + return; + } + if (!value || typeof value !== "object" || ancestors.has(value)) { + return; + } + ancestors.add(value); + for (const item of Array.isArray(value) ? value : Object.values(value)) { + collectStrings(item, strings, ancestors); + } + ancestors.delete(value); +}; + +// Takes the parsed file, so YAML comments are already gone and anchors are +// expanded into the services that use them. +export const extractComposeVariables = (spec: unknown): ComposeVariable[] => { + const found = new Map< + string, + { defaults: string[]; required: boolean; services: Set } + >(); + + const record = (value: unknown, service: string | null) => { + const strings: string[] = []; + collectStrings(value, strings); + for (const text of strings) { + for (const { name, operator, value: operand } of findVariableReferences( + text, + )) { + const entry = found.get(name) ?? { + defaults: [], + required: false, + services: new Set(), + }; + found.set(name, entry); + if (service) { + entry.services.add(service); + } + if (operator === ":-" || operator === "-") { + if (!entry.defaults.includes(operand ?? "")) { + entry.defaults.push(operand ?? ""); + } + } else if (operator === ":?" || operator === "?") { + entry.required = true; + } + } + } + }; + + if (!spec || typeof spec !== "object" || Array.isArray(spec)) { + return []; + } + for (const [key, section] of Object.entries(spec)) { + // Extension fields reach services through anchors, which the parser + // has already expanded where they are used. + if (key.startsWith("x-")) { + continue; + } + if (key === "services" && section && typeof section === "object") { + for (const [service, definition] of Object.entries(section)) { + record(definition, service); + } + } else { + record(section, null); + } + } + + return [...found.entries()] + .filter(([name]) => !DOKPLOY_PROVIDED_VARIABLES.has(name)) + .map(([name, entry]) => ({ + name, + defaultValue: entry.defaults[0] ?? null, + required: entry.required, + conflictingDefaults: entry.defaults.length > 1, + services: [...entry.services], + secret: isSecretEnvName(name), + placeholder: entry.defaults.some((value) => + PLACEHOLDER_PATTERN.test(value), + ), + })); +};