secrets dump: error handling, --skip-errors, --start-from, --simple

README.md

@@ -199,3 +199,21 @@ Dump secrets from SSM parameter store. You can optionally filter params by name
 ```bash
 ecsctrl secrets dump -e production.env --filter "db_.*" secrets.yaml
 ```
+
+Secrets are appended to the output file as soon as each one is fetched, so an interrupted dump keeps what was already written. By default the dump aborts on the first parameter that fails to be fetched (e.g. access denied); secrets written before the failure stay in the file. Use `--skip-errors` to skip such parameters instead - each skipped parameter is reported on stderr.
+
+```bash
+ecsctrl secrets dump -e production.env --skip-errors secrets.yaml
+```
+
+Use `--start-from` to dump only parameters whose names are alphabetically equal to or after the given value (e.g. to resume after a failure). It can be combined with `--filter`.
+
+```bash
+ecsctrl secrets dump -e production.env --start-from "/prod/db" secrets.yaml
+```
+
+By default only `SecureString` parameters without a description are written as `name: value`; the others include their `Type` and `Description`. Use `--simple` to write every parameter as `name: value`. Note that the type and description are not kept, so storing such a file creates all parameters as `SecureString`s.
+
+```bash
+ecsctrl secrets dump -e production.env --simple secrets.yaml
+```

ecsctrl/cli.py

@@ -3,12 +3,13 @@
 from email.policy import default
 
 import click
+from botocore.exceptions import BotoCoreError, ClientError
 
 from ecsctrl.loader import VarsLoader
 
 from .boto_client import BotoClient
 from .dump import generate_var_lut
-from .dump.secrets import dump_secrets, render_dumped_secrets
+from .dump.secrets import SecretFetchError, dump_secrets, render_dumped_secrets
 from .service_updater import ServiceUpdater, TaskDefinitionServiceUpdater, WaitForUpdate
 from .yaml_converter import (
     JOB_DEFINITION,
@@ -330,12 +331,33 @@ def store(
     default=None,
     help="Export only secrets matching given regexp pattern",
 )
+@click.option(
+    "--start-from",
+    type=str,
+    default=None,
+    help="Export only secrets whose names are alphabetically equal to or after given value",
+)
+@click.option(
+    "--simple",
+    is_flag=True,
+    default=False,
+    help="Write every secret as `key: value`, omitting its type and description",
+)
+@click.option(
+    "--skip-errors",
+    is_flag=True,
+    default=False,
+    help="Skip secrets that fail to be fetched (reported on stderr) instead of aborting",
+)
 @common_options
 @click.pass_context
 def dump(
     ctx,
     spec_file,
     filter,
+    start_from,
+    simple,
+    skip_errors,
     env_file,
     json_file,
     var,
@@ -345,8 +367,23 @@ def dump(
     vars = VarsLoader(env_file, var, json_file, sys_env).load()
     var_lut = generate_var_lut(vars)
     ssm = BotoClient("ssm", dry_run=ctx.obj["boto_client"].dry_run)
-    secrets = dump_secrets(ssm, filter)
-    render_dumped_secrets(click, secrets, var_lut, spec_file)
+
+    def on_error(error):
+        click.echo(f"⚠️ Skipping {error.name}: {error.cause}", err=True)
+
+    secrets = dump_secrets(
+        ssm, filter, on_error if skip_errors else None, start_from=start_from
+    )
+    try:
+        render_dumped_secrets(click, secrets, var_lut, spec_file, simple)
+    except SecretFetchError as e:
+        raise click.ClickException(
+            f"{e.name}: {e.cause}\n"
+            f"Secrets fetched before the failure were already written to {spec_file}. "
+            "Use --skip-errors to skip secrets that fail to be fetched."
+        )
+    except (BotoCoreError, ClientError) as e:
+        raise click.ClickException(f"Failed to list secrets: {e}")
 
 
 @service.command()

ecsctrl/dump/secrets.py

@@ -1,4 +1,5 @@
 import re
+from botocore.exceptions import BotoCoreError, ClientError
 from . import substitute_with_expressions
 from dataclasses import dataclass
 from typing import Generator, Optional, Sequence
@@ -28,15 +29,33 @@ def list_secrets(ssm):
             yield parameter
 
 
-def dump_secrets(ssm, filter=None) -> Generator[Parameter, None, None]:
+class SecretFetchError(Exception):
+    def __init__(self, name, cause):
+        super().__init__(f"Failed to fetch {name}: {cause}")
+        self.name = name
+        self.cause = cause
+
+
+def dump_secrets(
+    ssm, filter=None, on_error=None, start_from=None
+) -> Generator[Parameter, None, None]:
     for parameter in list_secrets(ssm):
         parameter_name = parameter["Name"]
+        if start_from is not None and parameter_name < start_from:
+            continue
         if filter is None or re.match(filter, parameter_name):
-            response = ssm.call(
-                "get_parameter",
-                Name=parameter_name,
-                WithDecryption=True,
-            )
+            try:
+                response = ssm.call(
+                    "get_parameter",
+                    Name=parameter_name,
+                    WithDecryption=True,
+                )
+            except (BotoCoreError, ClientError) as e:
+                error = SecretFetchError(parameter_name, e)
+                if on_error is None:
+                    raise error from e
+                on_error(error)
+                continue
 
             yield Parameter(
                 name=parameter_name,
@@ -46,20 +65,30 @@ def dump_secrets(ssm, filter=None) -> Generator[Parameter, None, None]:
             )
 
 
-def render_dumped_secrets(click, secrets: Sequence[Parameter], vars_lut, target_file):
-    with open(target_file, "w") as f:
+def render_dumped_secrets(
+    click, secrets: Sequence[Parameter], vars_lut, target_file, simple=False
+):
+    f = None
+    try:
         for parameter in secrets:
-            key, secret_text = rended_single_secret(parameter, vars_lut)
-            f.write(f"# Dumped from `{parameter.name}`:\n")
-            f.write(secret_text)
-            f.write(f"\n")
+            key, secret_text = rended_single_secret(parameter, vars_lut, simple)
+            if f is None:
+                f = open(target_file, "w")
+            f.write(f"# Dumped from `{parameter.name}`:\n{secret_text}\n")
+            f.flush()
             click.echo(f"🔑 Dumped secret {parameter.name} as {key}.")
 
+        if f is None:
+            open(target_file, "w").close()
+    finally:
+        if f is not None:
+            f.close()
+
 
-def rended_single_secret(parameter: Parameter, vars_lut):
+def rended_single_secret(parameter: Parameter, vars_lut, simple=False):
     name = parameter.name
     key = substitute_with_expressions(name, vars_lut)
-    if parameter.type == "SecureString" and parameter.description is None:
+    if simple or (parameter.type == "SecureString" and parameter.description is None):
         secret_text = render_simple_secret(key, parameter)
     else:
         secret_text = render_complex_secret(key, parameter)