diff --git a/apps/dokploy/__test__/compose/compose-env-merge.test.ts b/apps/dokploy/__test__/compose/compose-env-merge.test.ts new file mode 100644 index 00000000000..e58139a2996 --- /dev/null +++ b/apps/dokploy/__test__/compose/compose-env-merge.test.ts @@ -0,0 +1,167 @@ +import { parse } from "dotenv"; +import { describe, expect, it } from "vitest"; +import { + formatEnvValue, + mergeComposeEnv, + readComposeEnvState, +} from "@/lib/compose-env"; + +const HEADER = ["Defaults from ./docker-compose.yml (branch main)."]; + +describe("formatEnvValue", () => { + it.each([ + ["8080", "8080"], + ["", ""], + ["http://origin:80", "http://origin:80"], + ["a b", "'a b'"], + ["color #fff", "'color #fff'"], + ["it's", `"it's"`], + [`it's "quoted"`, `\`it's "quoted"\``], + ["C:\\path's", "`C:\\path's`"], + ])("writes %j so dotenv reads it back unchanged", (value, formatted) => { + expect(formatEnvValue(value)).toBe(formatted); + expect(parse(`KEY=${formatted}`).KEY).toBe(value); + }); +}); + +describe("readComposeEnvState", () => { + it("tells set, listed and absent variables apart", () => { + const stateOf = readComposeEnvState( + [ + "SET=1", + "export EXPORTED=2", + "# LISTED=3", + "# BOTH=old", + "BOTH=new", + "# a note about PROSE = text", + ].join("\n"), + ); + + expect(stateOf("SET")).toEqual({ kind: "set" }); + expect(stateOf("EXPORTED")).toEqual({ kind: "set" }); + expect(stateOf("LISTED")).toEqual({ kind: "listed", value: "3" }); + expect(stateOf("BOTH")).toEqual({ kind: "set" }); + expect(stateOf("PROSE")).toEqual({ kind: "absent" }); + expect(stateOf("MISSING")).toEqual({ kind: "absent" }); + }); +}); + +describe("mergeComposeEnv", () => { + it("adds commented defaults that dotenv does not apply", () => { + const merged = mergeComposeEnv( + "", + [ + { name: "PORT", value: "8080", services: ["web"] }, + { name: "MODE", value: "off", services: ["web"] }, + ], + { active: false, header: HEADER }, + ); + + expect(merged).toBe( + [ + "# Defaults from ./docker-compose.yml (branch main).", + "# PORT=8080", + "# MODE=off", + "", + ].join("\n"), + ); + expect(parse(merged)).toEqual({}); + }); + + it("adds active lines that dotenv applies", () => { + const merged = mergeComposeEnv( + "", + [{ name: "MODE", value: "a b", services: ["web"] }], + { active: true, header: HEADER }, + ); + + expect(parse(merged)).toEqual({ MODE: "a b" }); + }); + + it("never changes a variable that is already set", () => { + const merged = mergeComposeEnv( + "PORT=9000\nSECRET=mine", + [ + { name: "PORT", value: "8080", services: ["web"] }, + { name: "SECRET", value: "change-me", services: ["web"] }, + ], + { active: true, header: HEADER }, + ); + + expect(merged).toBe("PORT=9000\nSECRET=mine"); + }); + + it("refreshes commented defaults in place instead of adding them again", () => { + const env = "KEEP=1\n# LIMIT=60\n# my own note\nOTHER=2\n"; + + expect( + mergeComposeEnv( + env, + [{ name: "LIMIT", value: "100", services: ["engine"] }], + { active: false, header: HEADER }, + ), + ).toBe("KEEP=1\n# LIMIT=100\n# my own note\nOTHER=2\n"); + + expect( + mergeComposeEnv( + env, + [{ name: "LIMIT", value: "100", services: ["engine"] }], + { active: true, header: HEADER }, + ), + ).toBe("KEEP=1\nLIMIT=100\n# my own note\nOTHER=2\n"); + }); + + it("appends after the existing text and groups lines by service", () => { + const merged = mergeComposeEnv( + "KEEP=1\n\n", + [ + { name: "HTTP_PORT", value: "8080", services: ["caddy"] }, + { + name: "REDIS_URL", + value: "redis://redis:6379", + services: ["engine"], + }, + { name: "PATHS", value: "/hook", services: ["caddy", "engine"] }, + { name: "TLS", value: "off", services: ["caddy"] }, + { name: "NETWORK", value: "edge", services: [] }, + ], + { + active: false, + header: [...HEADER, "Remove the # to override a default."], + }, + ); + + expect(merged).toBe( + [ + "KEEP=1", + "", + "# Defaults from ./docker-compose.yml (branch main).", + "# Remove the # to override a default.", + "", + "# Used by caddy", + "# HTTP_PORT=8080", + "# TLS=off", + "", + "# Used by engine", + "# REDIS_URL=redis://redis:6379", + "", + "# Used by caddy, engine", + "# PATHS=/hook", + "", + "# Not tied to a service", + "# NETWORK=edge", + "", + ].join("\n"), + ); + expect(parse(merged)).toEqual({ KEEP: "1" }); + }); + + it("leaves the text alone when nothing is left to add", () => { + expect( + mergeComposeEnv("A=1", [{ name: "A", value: "2", services: [] }], { + active: false, + header: HEADER, + }), + ).toBe("A=1"); + }); +}); diff --git a/apps/dokploy/__test__/compose/compose-variables.test.ts b/apps/dokploy/__test__/compose/compose-variables.test.ts new file mode 100644 index 00000000000..756a5cb5088 --- /dev/null +++ b/apps/dokploy/__test__/compose/compose-variables.test.ts @@ -0,0 +1,225 @@ +import { + extractComposeVariables, + findVariableReferences, +} from "@dokploy/server/utils/docker/compose-variables"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const variablesOf = (yaml: string) => + extractComposeVariables(parse(yaml, { maxAliasCount: 10000 })); + +describe("findVariableReferences", () => { + it("reads every interpolation form compose supports", () => { + expect( + findVariableReferences( + "${A} ${B:-one} ${C-two} ${D:?need D} ${E?need E} ${F:+on} $G", + ), + ).toEqual([ + { name: "A", operator: null, value: null }, + { name: "B", operator: ":-", value: "one" }, + { name: "C", operator: "-", value: "two" }, + { name: "D", operator: ":?", value: "need D" }, + { name: "E", operator: "?", value: "need E" }, + { name: "F", operator: ":+", value: "on" }, + { name: "G", operator: null, value: null }, + ]); + }); + + it("treats $$ as a literal dollar sign", () => { + expect(findVariableReferences("echo $$HOME $${NOT_A_VAR} ${REAL}")).toEqual( + [{ name: "REAL", operator: null, value: null }], + ); + }); + + it("keeps nested defaults whole and reads the variables inside them", () => { + expect( + findVariableReferences("${URL:-http://${HOST:-localhost}:80}"), + ).toEqual([ + { + name: "URL", + operator: ":-", + value: "http://${HOST:-localhost}:80", + }, + { name: "HOST", operator: ":-", value: "localhost" }, + ]); + }); + + it("ignores text that only looks like a reference", () => { + expect(findVariableReferences("$ ${ ${1A} ${UNCLOSED:-x ${A B}")).toEqual( + [], + ); + }); +}); + +describe("extractComposeVariables", () => { + it("lists variables with their defaults and the services using them", () => { + const variables = variablesOf(` +services: + caddy: + image: caddy + ports: + - "\${HTTP_PORT:-8080}:80" + environment: + MODE: \${MODE:-off} + PATHS: \${PATHS:-/payments/webhook} + engine: + image: engine + environment: + PATHS: \${PATHS:-/payments/webhook} + TOKEN: \${ADMIN_TOKEN:-} + HOST: \${DB_HOST} + REQUIRED: \${DB_PASSWORD:?set a password} +`); + + expect(variables).toEqual([ + { + name: "HTTP_PORT", + defaultValue: "8080", + required: false, + conflictingDefaults: false, + services: ["caddy"], + secret: false, + placeholder: false, + }, + { + name: "MODE", + defaultValue: "off", + required: false, + conflictingDefaults: false, + services: ["caddy"], + secret: false, + placeholder: false, + }, + { + name: "PATHS", + defaultValue: "/payments/webhook", + required: false, + conflictingDefaults: false, + services: ["caddy", "engine"], + secret: false, + placeholder: false, + }, + { + name: "ADMIN_TOKEN", + defaultValue: "", + required: false, + conflictingDefaults: false, + services: ["engine"], + secret: true, + placeholder: false, + }, + { + name: "DB_HOST", + defaultValue: null, + required: false, + conflictingDefaults: false, + services: ["engine"], + secret: false, + placeholder: false, + }, + { + name: "DB_PASSWORD", + defaultValue: null, + required: true, + conflictingDefaults: false, + services: ["engine"], + secret: true, + placeholder: false, + }, + ]); + }); + + it("flags defaults that ask to be replaced", () => { + const variables = variablesOf(` +services: + engine: + environment: + SALT: \${HASH_SALT:-change-me-hash-salt} + ADMIN: \${CAP_ADMIN_KEY:-replace_me} + TTL: \${PASS_TTL_SECONDS:-1800} +`); + + expect( + variables.map(({ name, secret, placeholder }) => [ + name, + secret, + placeholder, + ]), + ).toEqual([ + ["HASH_SALT", false, true], + ["CAP_ADMIN_KEY", true, true], + ["PASS_TTL_SECONDS", true, false], + ]); + }); + + it("skips variables that appear only in YAML comments", () => { + const variables = variablesOf(` +services: + cap: + # The dashboard listens on 127.0.0.1:\${DASHBOARD_PORT} of the host. + ports: + - "127.0.0.1:\${DASHBOARD_PORT:-3100}:3000" + # \${ONLY_IN_A_COMMENT:-x} +`); + + expect( + variables.map(({ name, defaultValue }) => [name, defaultValue]), + ).toEqual([["DASHBOARD_PORT", "3100"]]); + }); + + it("follows anchors into the services that use them", () => { + const variables = variablesOf(` +x-common: &common + environment: + LOG_LEVEL: \${LOG_LEVEL:-info} +services: + api: + <<: *common + image: api + worker: + <<: *common + image: worker +`); + + expect(variables).toMatchObject([ + { name: "LOG_LEVEL", defaultValue: "info", services: ["api", "worker"] }, + ]); + }); + + it("flags a variable whose defaults differ between places", () => { + const variables = variablesOf(` +services: + a: + environment: + LIMIT: \${LIMIT:-60} + b: + environment: + LIMIT: \${LIMIT:-120} +`); + + expect(variables).toMatchObject([ + { name: "LIMIT", defaultValue: "60", conflictingDefaults: true }, + ]); + }); + + it("keeps variables outside services and leaves out the ones Dokploy sets", () => { + const variables = variablesOf(` +services: + web: + container_name: \${APP_NAME}-web + image: \${COMPOSE_PROJECT_NAME}:latest +networks: + edge: + name: \${EDGE_NETWORK:-edge} +`); + + expect(variables).toMatchObject([ + { name: "EDGE_NETWORK", defaultValue: "edge", services: [] }, + ]); + }); + + it("returns nothing for an empty or non-mapping file", () => { + expect(extractComposeVariables(null)).toEqual([]); + expect(extractComposeVariables(["${A}"])).toEqual([]); + }); +}); diff --git a/apps/dokploy/__test__/security/api-key-output-guard.test.ts b/apps/dokploy/__test__/security/api-key-output-guard.test.ts index 68a2381cfa0..ffab871c21f 100644 --- a/apps/dokploy/__test__/security/api-key-output-guard.test.ts +++ b/apps/dokploy/__test__/security/api-key-output-guard.test.ts @@ -69,6 +69,7 @@ const { restoreApiKeyText } = await import( const testRouter = createTRPCRouter({ compose: createTRPCRouter({ + loadComposeVariables: secretReader, one: protectedProcedure.query(() => ({ composeId: "compose-1", composeFile: storedCompose, @@ -263,6 +264,10 @@ describe("procedures that return raw secrets", () => { () => apiKey().application.revealEnvironment(), ], ["postgres.revealEnvironment", () => apiKey().postgres.revealEnvironment()], + [ + "compose.loadComposeVariables", + () => apiKey().compose.loadComposeVariables(), + ], ["docker.getConfig", () => apiKey().docker.getConfig()], ["docker.readContainerFile", () => apiKey().docker.readContainerFile()], [ diff --git a/apps/dokploy/__test__/security/service-environment-reveal.test.ts b/apps/dokploy/__test__/security/service-environment-reveal.test.ts index 19292261dbe..4cf5efd7b73 100644 --- a/apps/dokploy/__test__/security/service-environment-reveal.test.ts +++ b/apps/dokploy/__test__/security/service-environment-reveal.test.ts @@ -36,6 +36,7 @@ const serverMocks = vi.hoisted(() => ({ getContainerLogs: vi.fn(), getContainersByAppNameMatch: vi.fn(), getWebServerSettings: vi.fn(), + loadComposeVariables: vi.fn(), loadServices: vi.fn(), mechanizeDockerContainer: vi.fn(), randomizeComposeFile: vi.fn(), @@ -114,6 +115,7 @@ vi.mock("@dokploy/server", () => ({ getContainerLogs: serverMocks.getContainerLogs, getContainersByAppNameMatch: serverMocks.getContainersByAppNameMatch, getWebServerSettings: serverMocks.getWebServerSettings, + loadComposeVariables: serverMocks.loadComposeVariables, loadServices: serverMocks.loadServices, mechanizeDockerContainer: serverMocks.mechanizeDockerContainer, randomizeComposeFile: serverMocks.randomizeComposeFile, @@ -337,6 +339,26 @@ describe("service environment reveal boundary", () => { }); }); + it("reads compose variables only for callers that can read env vars", async () => { + permissionMocks.checkServicePermissionAndAccess.mockRejectedValueOnce( + new TRPCError({ code: "UNAUTHORIZED", message: "denied" }), + ); + + await expect( + composeRouter + .createCaller(createContext()) + .loadComposeVariables({ composeId: "compose-1" }), + ).rejects.toMatchObject({ code: "UNAUTHORIZED" }); + + expect( + permissionMocks.checkServicePermissionAndAccess, + ).toHaveBeenCalledWith(expect.anything(), "compose-1", { + service: ["create"], + envVars: ["read"], + }); + expect(serverMocks.loadComposeVariables).not.toHaveBeenCalled(); + }); + it("keeps the compose file for callers that can read env vars", async () => { permissionMocks.hasPermission.mockResolvedValueOnce(true); diff --git a/apps/dokploy/components/dashboard/application/environment/show-environment.tsx b/apps/dokploy/components/dashboard/application/environment/show-environment.tsx index cd0bd414d0f..c59b5239208 100644 --- a/apps/dokploy/components/dashboard/application/environment/show-environment.tsx +++ b/apps/dokploy/components/dashboard/application/environment/show-environment.tsx @@ -4,6 +4,10 @@ import { type CSSProperties, useEffect, useState } from "react"; import { useForm } from "react-hook-form"; import { toast } from "sonner"; import { z } from "zod"; +import { + type ComposeEnvImport, + ComposeVariablesDialog, +} from "@/components/dashboard/compose/environment/compose-variables-dialog"; import { CodeEditor } from "@/components/shared/code-editor"; import { useEnvCompletionSource } from "@/components/shared/env-autocomplete"; import { VaultImportDialog } from "@/components/shared/vault-import-dialog"; @@ -26,6 +30,7 @@ import { } from "@/components/ui/form"; import { Switch } from "@/components/ui/switch"; import { Toggle } from "@/components/ui/toggle"; +import { mergeComposeEnv } from "@/lib/compose-env"; import { api } from "@/utils/api"; import type { ServiceType } from "../advanced/show-resources"; @@ -44,6 +49,11 @@ interface Props { export const ShowEnvironment = ({ id, type }: Props) => { const { data: permissions } = api.user.getPermissions.useQuery(); const canWrite = permissions?.envVars.write ?? false; + const canImportComposeVariables = + type === "compose" && + canWrite && + (permissions?.envVars.read ?? false) && + (permissions?.service.create ?? false); const queryMap = { compose: () => api.compose.one.useQuery({ composeId: id }, { enabled: !!id }), @@ -192,6 +202,38 @@ export const ShowEnvironment = ({ id, type }: Props) => { } }; + // Imported lines go into the revealed text: a masked editor cannot be + // revealed again while it holds unsaved changes. + const revealForEditing = async () => { + if (isEnvVisible) { + try { + await handleReveal(); + } catch { + return false; + } + setIsEnvVisible(false); + } + return true; + }; + + const handleComposeImport = ({ + entries, + active, + header, + }: ComposeEnvImport) => { + form.setValue( + "environment", + mergeComposeEnv(form.getValues("environment"), entries, { + active, + header, + }), + { shouldDirty: true }, + ); + toast.success( + `Added ${entries.length} variable${entries.length === 1 ? "" : "s"}. Review them and click Save.`, + ); + }; + // Add keyboard shortcut for Ctrl+S/Cmd+S useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { @@ -257,7 +299,16 @@ export const ShowEnvironment = ({ id, type }: Props) => { name="environment" render={({ field }) => ( -
+
+ {canImportComposeVariables && ( + + )} Promise; + onImport: (changes: ComposeEnvImport) => void; +} + +const compareWithEnv = ( + stateOf: ReturnType, + name: string, + value: string, +) => { + const state = stateOf(name); + const listedValue = state.kind === "listed" ? state.value.trim() : null; + return { + state: state.kind, + listedValue, + changed: listedValue !== null && listedValue !== formatEnvValue(value), + }; +}; + +export const ComposeVariablesDialog = ({ + composeId, + currentEnv, + createEnvFile, + beforeOpen, + onImport, +}: Props) => { + const [isOpen, setIsOpen] = useState(false); + const [isPreparing, setIsPreparing] = useState(false); + const [filter, setFilter] = useState(""); + const [selected, setSelected] = useState>(new Set()); + const [active, setActive] = useState(false); + const { mutateAsync, data, error, isPending, reset } = + api.compose.loadComposeVariables.useMutation(); + // beforeOpen may reveal the environment, which arrives after load() began. + const currentEnvRef = useRef(currentEnv); + currentEnvRef.current = currentEnv; + + const rows = useMemo(() => { + const stateOf = readComposeEnvState(currentEnv); + return (data?.variables ?? []).map((variable) => { + const value = variable.defaultValue ?? ""; + return { + ...variable, + value, + ...compareWithEnv(stateOf, variable.name, value), + }; + }); + }, [data, currentEnv]); + + const visibleRows = useMemo(() => { + const query = filter.trim().toLowerCase(); + if (!query) { + return rows; + } + return rows.filter( + (row) => + row.name.toLowerCase().includes(query) || + row.services.some((service) => service.toLowerCase().includes(query)), + ); + }, [rows, filter]); + + const selectableRows = visibleRows.filter((row) => row.state !== "set"); + const setCount = rows.filter((row) => row.state === "set").length; + + const load = async () => { + setSelected(new Set()); + try { + const result = await mutateAsync({ composeId }); + const stateOf = readComposeEnvState(currentEnvRef.current); + setSelected( + new Set( + result.variables + .filter((variable) => { + const { state, changed } = compareWithEnv( + stateOf, + variable.name, + variable.defaultValue ?? "", + ); + return state === "absent" || changed; + }) + .map((variable) => variable.name), + ), + ); + } catch { + // The error is shown in the dialog. + } + }; + + const handleOpenChange = async (open: boolean) => { + if (!open) { + setIsOpen(false); + setFilter(""); + reset(); + return; + } + setIsPreparing(true); + try { + if (!(await beforeOpen())) { + return; + } + } finally { + setIsPreparing(false); + } + setIsOpen(true); + await load(); + }; + + const toggleRow = (name: string, checked: boolean) => { + setSelected((previous) => { + const next = new Set(previous); + if (checked) { + next.add(name); + } else { + next.delete(name); + } + return next; + }); + }; + + const allVisibleSelected = + selectableRows.length > 0 && + selectableRows.every((row) => selected.has(row.name)); + + const toggleAllVisible = () => { + setSelected((previous) => { + const next = new Set(previous); + for (const row of selectableRows) { + if (allVisibleSelected) { + next.delete(row.name); + } else { + next.add(row.name); + } + } + return next; + }); + }; + + const handleImport = () => { + if (!data) { + return; + } + const source = `${data.composePath}${data.branch ? ` (branch ${data.branch})` : ""}`; + onImport({ + entries: rows + .filter((row) => row.state !== "set" && selected.has(row.name)) + .map(({ name, value, services }) => ({ name, value, services })), + active, + header: active + ? [`Values copied from ${source}.`] + : [ + `Defaults from ${source}.`, + "Lines starting with # are not applied: remove the # and change the value to override a default.", + ], + }); + handleOpenChange(false); + }; + + const selectedCount = rows.filter( + (row) => row.state !== "set" && selected.has(row.name), + ).length; + + return ( + + + + + Variables from the compose file + + {data + ? `Variables used in ${data.composePath}${data.branch ? ` on branch ${data.branch}` : ""}, with the value the file falls back to while a variable is not set here.` + : "Fetching the compose file from the source selected in General."} + + + +
+ {!createEnvFile && ( + + Create Environment File is off, so variables set here do not reach + the compose file until it is turned on. + + )} + + {isPending || (!data && !error) ? ( +
+ + Fetching the compose file... +
+ ) : error ? ( +
+ {error.message} + +
+ ) : rows.length === 0 ? ( +
+ The compose file does not use any variables. +
+ ) : ( + <> +
+ {rows.length} variables: {setCount} set here,{" "} + {rows.length - setCount} use the default from the file. +
+ setFilter(event.target.value)} + /> +
+ + + {selectedCount} selected + +
+ +
+ {visibleRows.map((row) => ( +
+ + toggleRow(row.name, checked === true) + } + /> +
+
+ + {row.state === "set" && ( + set here + )} + {row.state === "listed" && !row.changed && ( + listed + )} + {row.changed && ( + default changed + )} + {row.secret && secret} + {row.placeholder && ( + placeholder + )} + {row.required && ( + required + )} + {row.conflictingDefaults && ( + defaults differ + )} +
+ + {row.defaultValue === null + ? "no default" + : row.defaultValue === "" + ? "empty by default" + : `default: ${row.defaultValue}`} + + {row.changed && ( + + listed here as: {row.listedValue} + + )} + {row.services.length > 0 && ( + + Used by {row.services.join(", ")} + + )} + {row.placeholder && ( + + The default from the file is a placeholder. Set your + own value. + + )} +
+
+ ))} +
+ + + )} + +
+
+ +

+ {active + ? "Values are copied as they are and override the file, so later changes to the defaults in the repository no longer apply." + : "Lines are added commented out (# NAME=value): the defaults from the file keep applying, and you override one by removing its #. Adding again refreshes the commented values."} +

+
+ +
+
+ + + + + + + ); +}; diff --git a/apps/dokploy/lib/compose-env.ts b/apps/dokploy/lib/compose-env.ts new file mode 100644 index 00000000000..c673d4110c1 --- /dev/null +++ b/apps/dokploy/lib/compose-env.ts @@ -0,0 +1,111 @@ +export interface ComposeEnvEntry { + name: string; + value: string; + services: string[]; +} + +export type ComposeEnvState = + | { kind: "set" } + | { kind: "listed"; value: string } + | { kind: "absent" }; + +const ACTIVE_LINE = /^\s*(?:export\s+)?([A-Za-z_][\w.-]*)\s*=/; +// Only `# NAME=value` with nothing between the name and "=", so prose in +// comments is not taken for a variable. +const COMMENTED_LINE = /^\s*#\s*([A-Za-z_]\w*)=(.*)$/; + +// dotenv ends an unquoted value at " #" and trims it, and expands escapes such +// as \n only inside double quotes. +export const formatEnvValue = (value: string) => { + if (!/[\s#"'`\\]/.test(value)) { + return value; + } + if (!value.includes("'")) { + return `'${value}'`; + } + if (!value.includes('"') && !value.includes("\\")) { + return `"${value}"`; + } + return `\`${value}\``; +}; + +const findCommentedLines = (lines: string[]) => { + const commented = new Map(); + lines.forEach((line, index) => { + const name = COMMENTED_LINE.exec(line)?.[1]; + if (name && !commented.has(name)) { + commented.set(name, index); + } + }); + return commented; +}; + +export const readComposeEnvState = (env: string) => { + const lines = env.split("\n"); + const active = new Set(); + for (const line of lines) { + const name = ACTIVE_LINE.exec(line)?.[1]; + if (name) { + active.add(name); + } + } + const commented = findCommentedLines(lines); + return (name: string): ComposeEnvState => { + if (active.has(name)) { + return { kind: "set" }; + } + const index = commented.get(name); + if (index !== undefined) { + return { + kind: "listed", + value: COMMENTED_LINE.exec(lines[index] ?? "")?.[2] ?? "", + }; + } + return { kind: "absent" }; + }; +}; + +// Variables already set stay as they are. A commented `# NAME=value` line is +// rewritten in place, so importing again refreshes the defaults it shows; +// the rest go to a block at the end, grouped by the services that use them. +export const mergeComposeEnv = ( + env: string, + entries: ComposeEnvEntry[], + { active, header }: { active: boolean; header: string[] }, +) => { + const lines = env.split("\n"); + const state = readComposeEnvState(env); + const commented = findCommentedLines(lines); + const groups = new Map(); + + for (const entry of entries) { + if (state(entry.name).kind === "set") { + continue; + } + const assignment = `${entry.name}=${formatEnvValue(entry.value)}`; + const line = active ? assignment : `# ${assignment}`; + const index = commented.get(entry.name); + if (index !== undefined) { + lines[index] = line; + continue; + } + const group = entry.services.join(", "); + groups.set(group, [...(groups.get(group) ?? []), line]); + } + + const merged = lines.join("\n"); + if (groups.size === 0) { + return merged; + } + + const block = header.map((line) => `# ${line}`); + for (const [group, groupLines] of groups) { + if (groups.size > 1) { + block.push("", group ? `# Used by ${group}` : "# Not tied to a service"); + } + block.push(...groupLines); + } + + const base = merged.trimEnd(); + return `${base}${base ? "\n\n" : ""}${block.join("\n")}\n`; +}; diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts index ce93c44a712..b4f8b31d664 100644 --- a/apps/dokploy/server/api/routers/compose.ts +++ b/apps/dokploy/server/api/routers/compose.ts @@ -23,6 +23,7 @@ import { getContainersByAppNameMatch, getWebServerSettings, IS_CLOUD, + loadComposeVariables, loadServices, markDeploymentOperationDispatched, randomizeComposeFile, @@ -549,6 +550,18 @@ export const composeRouter = createTRPCRouter({ }); return await loadServices(input.composeId, input.type); }), + loadComposeVariables: protectedProcedure + .input(apiFindCompose) + .mutation(async ({ input, ctx }) => { + // Defaults in a compose file are often secrets, so this needs the + // same access as reading the environment; fetching the source also + // needs what fetchSourceType needs. + await checkServicePermissionAndAccess(ctx, input.composeId, { + service: ["create"], + envVars: ["read"], + }); + return await loadComposeVariables(input.composeId); + }), loadMountsByService: protectedProcedure .input( z.object({ diff --git a/apps/dokploy/server/api/utils/api-key-output-guard.ts b/apps/dokploy/server/api/utils/api-key-output-guard.ts index 2fe60e7104e..5fccb147213 100644 --- a/apps/dokploy/server/api/utils/api-key-output-guard.ts +++ b/apps/dokploy/server/api/utils/api-key-output-guard.ts @@ -24,6 +24,7 @@ import { isApiKeySession } from "@/server/api/utils/super-session"; // make safe (a container file may be nothing but one token), so an API key // cannot call them at all. const API_KEY_DENIED_PROCEDURES = new Set([ + "compose.loadComposeVariables", "docker.getConfig", "docker.readContainerFile", "dockerVolume.readVolumeFile", diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index f2debf65233..3aeb3ec45a0 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -178,6 +178,7 @@ export const PROCEDURE_ACCESS: Record = { "compose.import": "write", "compose.isolatedDeployment": "write", "compose.killBuild": "write", + "compose.loadComposeVariables": "write", "compose.loadMountsByService": "read", "compose.loadServices": "read", "compose.move": "write", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 5f4d5b45f6f..9b3119c7ec4 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -118,6 +118,7 @@ export * from "./utils/docker/compose/network"; export * from "./utils/docker/compose/secrets"; export * from "./utils/docker/compose/service"; export * from "./utils/docker/compose/volume"; +export * from "./utils/docker/compose-variables"; export * from "./utils/docker/domain"; export * from "./utils/docker/types"; export * from "./utils/docker/utils"; diff --git a/packages/server/src/services/compose.ts b/packages/server/src/services/compose.ts index e03288ef41c..608b98b8788 100644 --- a/packages/server/src/services/compose.ts +++ b/packages/server/src/services/compose.ts @@ -10,6 +10,7 @@ import { } from "@dokploy/server/db/schema"; import { getBuildComposeCommand } from "@dokploy/server/utils/builders/compose"; import { randomizeSpecificationFile } from "@dokploy/server/utils/docker/compose"; +import { extractComposeVariables } from "@dokploy/server/utils/docker/compose-variables"; import { cloneCompose, loadDockerCompose, @@ -235,6 +236,77 @@ export const loadServices = async ( return [...services]; }; +const getComposeBranch = (compose: Compose) => { + switch (compose.sourceType) { + case "github": + return compose.branch; + case "gitlab": + return compose.gitlabBranch; + case "bitbucket": + return compose.bitbucketBranch; + case "gitea": + return compose.giteaBranch; + case "git": + return compose.customGitBranch; + default: + return null; + } +}; + +export const loadComposeVariables = async (composeId: string) => { + const compose = await findComposeById(composeId); + + // Fetching replaces the code directory that a running build reads from. + if (compose.composeStatus === "running") { + throw new TRPCError({ + code: "CONFLICT", + message: "A deployment is running. Try again when it finishes.", + }); + } + + try { + const command = await cloneCompose(compose); + if (compose.serverId) { + await execAsyncRemote(compose.serverId, command); + } else { + await execAsync(command); + } + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Could not fetch the source of this compose service", + cause: error, + }); + } + + let composeData: ComposeSpecification | null; + try { + composeData = compose.serverId + ? await loadDockerComposeRemote(compose) + : await loadDockerCompose(compose); + } catch (error) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "The compose file is not valid YAML", + cause: error, + }); + } + + if (!composeData) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Compose file not found", + }); + } + + return { + composePath: + compose.sourceType === "raw" ? "docker-compose.yml" : compose.composePath, + branch: getComposeBranch(compose), + variables: extractComposeVariables(composeData), + }; +}; + export const updateCompose = async ( composeId: string, composeData: Partial, diff --git a/packages/server/src/utils/docker/compose-variables.ts b/packages/server/src/utils/docker/compose-variables.ts new file mode 100644 index 00000000000..82d1c46831f --- /dev/null +++ b/packages/server/src/utils/docker/compose-variables.ts @@ -0,0 +1,181 @@ +import { isSecretEnvName } from "../env-upsert"; + +export interface ComposeVariableReference { + name: string; + operator: string | null; + value: string | null; +} + +export interface ComposeVariable { + name: string; + // null when the file gives no default, so the variable is empty unless set. + defaultValue: string | null; + // `${NAME:?message}`: compose refuses to start while the variable is unset. + required: boolean; + // Different places in the file fall back to different values. + conflictingDefaults: boolean; + services: string[]; + secret: boolean; + // The default asks to be replaced, such as `change-me-salt`. + placeholder: boolean; +} + +// Dokploy writes these into the .env file of every compose service itself. +const DOKPLOY_PROVIDED_VARIABLES = new Set([ + "APP_NAME", + "COMPOSE_PROJECT_NAME", + "COMPOSE_PREFIX", + "DOCKER_CONFIG", +]); + +const PLACEHOLDER_PATTERN = /change[-_]?me|replace[-_]?me/i; + +const NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*/; +const OPERATOR_PATTERN = /^:?[-?+]/; + +const findClosingBrace = (text: string, start: number) => { + let depth = 1; + let index = start; + while (index < text.length) { + if (text[index] === "$" && text[index + 1] === "$") { + index += 2; + continue; + } + if (text[index] === "$" && text[index + 1] === "{") { + depth++; + index += 2; + continue; + } + if (text[index] === "}" && --depth === 0) { + return index; + } + index++; + } + return -1; +}; + +// Follows compose interpolation: `$$` is a literal dollar sign, and a value +// after an operator may hold further references (`${A:-${B:-x}}`). +export const findVariableReferences = ( + text: string, +): ComposeVariableReference[] => { + const references: ComposeVariableReference[] = []; + let index = text.indexOf("$"); + while (index !== -1) { + const next = text[index + 1]; + let resumeAt = index + 1; + if (next === "$") { + resumeAt = index + 2; + } else if (next === "{") { + const name = NAME_PATTERN.exec(text.slice(index + 2))?.[0]; + const afterName = index + 2 + (name?.length ?? 0); + const operator = + OPERATOR_PATTERN.exec(text.slice(afterName))?.[0] ?? null; + const valueStart = afterName + (operator?.length ?? 0); + const end = name ? findClosingBrace(text, valueStart) : -1; + if (name && end !== -1 && (operator || end === valueStart)) { + const value = operator ? text.slice(valueStart, end) : null; + references.push({ name, operator, value }); + if (value) { + references.push(...findVariableReferences(value)); + } + resumeAt = end + 1; + } + } else { + const name = NAME_PATTERN.exec(text.slice(index + 1))?.[0]; + if (name) { + references.push({ name, operator: null, value: null }); + resumeAt = index + 1 + name.length; + } + } + index = text.indexOf("$", resumeAt); + } + return references; +}; + +const collectStrings = ( + value: unknown, + strings: string[], + ancestors = new Set(), +) => { + if (typeof value === "string") { + strings.push(value); + return; + } + if (!value || typeof value !== "object" || ancestors.has(value)) { + return; + } + ancestors.add(value); + for (const item of Array.isArray(value) ? value : Object.values(value)) { + collectStrings(item, strings, ancestors); + } + ancestors.delete(value); +}; + +// Takes the parsed file, so YAML comments are already gone and anchors are +// expanded into the services that use them. +export const extractComposeVariables = (spec: unknown): ComposeVariable[] => { + const found = new Map< + string, + { defaults: string[]; required: boolean; services: Set } + >(); + + const record = (value: unknown, service: string | null) => { + const strings: string[] = []; + collectStrings(value, strings); + for (const text of strings) { + for (const { name, operator, value: operand } of findVariableReferences( + text, + )) { + const entry = found.get(name) ?? { + defaults: [], + required: false, + services: new Set(), + }; + found.set(name, entry); + if (service) { + entry.services.add(service); + } + if (operator === ":-" || operator === "-") { + if (!entry.defaults.includes(operand ?? "")) { + entry.defaults.push(operand ?? ""); + } + } else if (operator === ":?" || operator === "?") { + entry.required = true; + } + } + } + }; + + if (!spec || typeof spec !== "object" || Array.isArray(spec)) { + return []; + } + for (const [key, section] of Object.entries(spec)) { + // Extension fields reach services through anchors, which the parser + // has already expanded where they are used. + if (key.startsWith("x-")) { + continue; + } + if (key === "services" && section && typeof section === "object") { + for (const [service, definition] of Object.entries(section)) { + record(definition, service); + } + } else { + record(section, null); + } + } + + return [...found.entries()] + .filter(([name]) => !DOKPLOY_PROVIDED_VARIABLES.has(name)) + .map(([name, entry]) => ({ + name, + defaultValue: entry.defaults[0] ?? null, + required: entry.required, + conflictingDefaults: entry.defaults.length > 1, + services: [...entry.services], + secret: isSecretEnvName(name), + placeholder: entry.defaults.some((value) => + PLACEHOLDER_PATTERN.test(value), + ), + })); +};