diff --git a/.gitignore b/.gitignore index deaa14d1..fe9db2ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env .envrc .direnv/ +__pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55221b34..174b5f95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_language_version: node: system -exclude: '(stacks/_templates/minio-.*/rendered-chart\.yaml|\.svg)$' +exclude: '\.(ipynb|svg)$' repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/.scripts/render_drawio_images.py b/.scripts/render_drawio_images.py new file mode 100755 index 00000000..d110efdd --- /dev/null +++ b/.scripts/render_drawio_images.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python +"""Re-render the *.drawio.png overview diagrams, keeping them editable. + +draw.io shapes can reference logos either by URL (image=https://...) or by +embedded data URI (image=data:image/png,). Remote URLs are not fetched +by the renderer when exporting to PNG, so those logos come out blank. For each +diagram this script reads the source out of the PNG's mxfile textual chunk, +rewrites every remote reference into a data URI, and re-exports the PNG with +the source embedded again. + +Requires the draw.io desktop CLI, which nixpkgs marks as unfree. On NixOS: + NIXPKGS_ALLOW_UNFREE=1 nix-shell -p drawio-headless \ + --run .scripts/render_drawio_images.py + +Usage: + render_drawio_images.py [FILE...] re-render (default: all of docs/) + render_drawio_images.py --check [FILE...] report remote references only +""" + +import argparse +import base64 +import os +import re +import shutil +import struct +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +import zlib +from pathlib import Path + +USER_AGENT = "stackable-demos-drawio-inliner/1.0 (+https://github.com/stackabletech/demos)" +TIMEOUT = 30 + +# draw.io writes data URIs without the ";base64" marker, e.g. +# "data:image/png,iVBORw0...". Match that convention so round-tripping through +# the draw.io editor produces no spurious diffs. +EXTENSION_MIME = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".svg": "image/svg+xml", + ".webp": "image/webp", +} + +IMAGE_URL_RE = re.compile(r"(?<=\bimage=)(https?://[^;]+)") + + +def decompress(data: bytes) -> bytes: + """Inflate a PNG textual chunk payload. + + The spec says this is a zlib datastream, but draw.io writes a bare deflate + stream with no zlib header, so fall back to that. + """ + try: + return zlib.decompress(data) + except zlib.error: + return zlib.decompress(data, -zlib.MAX_WBITS) + + +def read_png_diagram(path: Path) -> str: + """Return the diagram XML from a draw.io PNG's mxfile textual chunk. + + The VS Code extension stores the diagram uncompressed in a tEXt chunk, + while the desktop CLI's --embed-diagram compresses it into a zTXt, so both + have to be understood for a re-render to be repeatable. + """ + data = path.read_bytes() + if data[:8] != b"\x89PNG\r\n\x1a\n": + raise ValueError(f"{path} is not a PNG") + offset = 8 + while offset < len(data): + (length,) = struct.unpack(">I", data[offset : offset + 4]) + chunk_type = data[offset + 4 : offset + 8] + payload = data[offset + 8 : offset + 8 + length] + if chunk_type in (b"tEXt", b"zTXt", b"iTXt"): + keyword, rest = payload.split(b"\x00", 1) + if keyword in (b"mxfile", b"mxGraphModel"): + if chunk_type == b"zTXt": + # Skip the compression-method byte. + rest = decompress(rest[1:]) + elif chunk_type == b"iTXt": + # Compression flag, method, then null-terminated language + # and translated-keyword fields before the text itself. + compressed = rest[0] + rest = rest[2:].split(b"\x00", 2)[2] + if compressed: + rest = decompress(rest) + return urllib.parse.unquote(rest.decode("latin1")) + offset += 12 + length + raise ValueError(f"{path} has no embedded draw.io diagram") + + +def read_diagram(path: Path) -> str: + if path.suffix.lower() == ".png": + return read_png_diagram(path) + return path.read_text(encoding="utf-8") + + +def inflate_diagram(text: str) -> str: + """Decompress a draw.io body, if it is compressed.""" + raw = base64.b64decode(text) + return urllib.parse.unquote(zlib.decompress(raw, -zlib.MAX_WBITS).decode("utf-8")) + + +def normalise(xml: str) -> str: + """Return the diagram XML with any compressed bodies expanded.""" + root = ET.fromstring(xml) + changed = False + for diagram in root.iter("diagram"): + # An uncompressed diagram holds an child element, and + # its text is just the whitespace in front of it. A compressed one has + # no children and carries a deflated, base64 payload as its text. + if len(diagram) == 0 and diagram.text and diagram.text.strip(): + model = ET.fromstring(inflate_diagram(diagram.text.strip())) + diagram.text = None + diagram.append(model) + changed = True + return ET.tostring(root, encoding="unicode") if changed else xml + + +def fetch_data_uri(url: str) -> str: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=TIMEOUT) as response: + payload = response.read() + mime = (response.headers.get_content_type() or "").lower() + if not mime.startswith("image/"): + mime = EXTENSION_MIME.get(Path(urllib.parse.urlparse(url).path).suffix.lower(), "") + if not mime: + raise ValueError(f"could not determine an image type for {url}") + return f"data:{mime},{base64.b64encode(payload).decode('ascii')}" + + +def remote_urls(xml: str) -> list[str]: + root = ET.fromstring(xml) + found = [] + for cell in root.iter(): + for url in IMAGE_URL_RE.findall(cell.get("style", "")): + if url not in found: + found.append(url) + return found + + +def inline(xml: str) -> tuple[str, list[str]]: + """Replace every remote image reference with a data URI. + + Returns the rewritten XML and the list of URLs that could not be fetched. + """ + root = ET.fromstring(xml) + cache: dict[str, str] = {} + failed: list[str] = [] + + def replace(match: re.Match[str]) -> str: + url = match.group(0) + if url in cache: + return cache[url] + if url in failed: + return url + try: + cache[url] = fetch_data_uri(url) + except Exception as error: # noqa: BLE001 - reported, not swallowed + print(f" FAILED {url}: {error}", file=sys.stderr) + failed.append(url) + return url + print(f" inlined {url} ({len(cache[url]) // 1024} KiB)", file=sys.stderr) + return cache[url] + + for cell in root.iter(): + style = cell.get("style") + if style: + updated = IMAGE_URL_RE.sub(replace, style) + if updated != style: + cell.set("style", updated) + + return ET.tostring(root, encoding="unicode"), failed + + +def export(drawio: str, source: Path, target: Path, scale: str, border: str) -> None: + """Render diagram XML to PNG, keeping the source embedded in the PNG.""" + # --embed-diagram writes the XML back into the PNG's mxfile tEXt chunk, so + # the exported file can still be opened and edited as a diagram. + subprocess.run( + [ + drawio, "--export", "--format", "png", "--embed-diagram", + "--scale", scale, "--border", border, + "--output", str(target), str(source), + ], + check=True, + ) + + +def default_inputs() -> list[Path]: + docs = Path(__file__).resolve().parent.parent / "docs" + return sorted(docs.rglob("*.drawio.png")) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "inputs", + nargs="*", + type=Path, + help="diagrams to process (default: every *.drawio.png under docs/)", + ) + parser.add_argument( + "--check", + action="store_true", + help="only report remote image references, do not fetch or re-render", + ) + parser.add_argument( + "--drawio", + default=os.environ.get("DRAWIO", "drawio"), + help="draw.io desktop CLI to render with (default: drawio)", + ) + # Match the scale/border draw.io recorded on the existing exports. + parser.add_argument("--scale", default="1") + parser.add_argument("--border", default="20") + args = parser.parse_args() + + inputs = args.inputs or default_inputs() + if not inputs: + print("no diagrams found", file=sys.stderr) + return 1 + + if args.check: + remaining = 0 + for path in inputs: + urls = remote_urls(normalise(read_diagram(path))) + remaining += len(urls) + print(f"{path}: {len(urls)} remote image(s)") + for url in urls: + print(f" {url}") + return 1 if remaining else 0 + + if shutil.which(args.drawio) is None: + print( + f"error: '{args.drawio}' not found. On NixOS, run inside 'nix-shell'.", + file=sys.stderr, + ) + return 1 + + failures = 0 + with tempfile.TemporaryDirectory() as workdir: + for index, path in enumerate(inputs): + print(f"==> {path}") + xml, failed = inline(normalise(read_diagram(path))) + failures += len(failed) + + # The diagrams are all called overview.drawio.png, so number the + # intermediate files to keep them apart. + source = Path(workdir) / f"{index}-{path.name.removesuffix('.png')}" + source.write_text(xml, encoding="utf-8") + export(args.drawio, source, path, args.scale, args.border) + + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/README.md b/README.md index 824c74f8..e7735b86 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,12 @@ A demo is an end-to-end demonstration of the usage of the Stackable data platform. It is tied to a specific stack of the Stackable data platform, which will provide the required products for the demo. The list of demos can be browsed either -via our documentation page [here][demo-overview] or with the `demo list` command of our `stackablectl` tool. More -information about this command can be found [here][demo-cmd]. +via our [documentation page][demo-overview] or with the `demo list` command of our `stackablectl` tool. More +information about this command can be found in our [stackablectl demo documentation][demo-cmd] page. -A stack is a collection of ready-to-use Stackable data products as well as required third-party services like Postgresql -or MinIO. It is tied to a specific release of the Stackable data platform, which will provide the required operators for -the stack. Stacks can be listed using the `stack list` command, see [here][stack-cmd] for more information. +A stack is a collection of ready-to-use Stackable data products as well as required third-party services like PostgreSQL +or Garage. It is tied to a specific release of the Stackable data platform, which will provide the required operators for +the stack. Stacks can be listed using the `stack list` command, see our [stackablectl stack documentation][stack-cmd] page for more information. [stack-cmd]: https://docs.stackable.tech/management/stable/stackablectl/commands/stack [demo-cmd]: https://docs.stackable.tech/management/stable/stackablectl/commands/demo diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index c3aed021..80aba8ae 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -19,8 +19,8 @@ spec: - pipefail - -c - | - echo 'Waiting for all minio instances to be ready' - kubectl wait --for=condition=ready --timeout=30m pod -l app=minio,release=minio,stackable.tech/vendor=Stackable -n {{ NAMESPACE }} + echo 'Waiting for Garage to be ready' + kubectl wait --for=condition=ready --timeout=30m pod -l app=garage -n {{ NAMESPACE }} echo 'Waiting for all kafka brokers to be ready' kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka,app.kubernetes.io/instance=kafka -n {{ NAMESPACE }} echo 'Waiting for all nifi instances to be ready' @@ -154,7 +154,7 @@ data: - org.apache.iceberg:iceberg-spark-runtime-4.1_2.13:1.11.0 - org.apache.spark:spark-sql-kafka-0-10_2.13:4.1.2 s3connection: - reference: minio + reference: garage sparkConf: spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions spark.sql.catalog.lakehouse: org.apache.iceberg.spark.SparkCatalog diff --git a/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml b/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml index a8e2a028..1299cf72 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml @@ -8,100 +8,137 @@ spec: spec: containers: - name: load-test-data - image: "bitnamilegacy/minio:2024-debian-12" - # Please try to order the load jobs from small to large datasets + image: docker.io/rclone/rclone:1.75.1 command: - - bash - - -xeuo + - /bin/sh + - -euo - pipefail - -c - | - # Copy the CA cert from the "tls" SecretClass - cp -v /etc/minio/mc/original_certs/ca.crt /.mc/certs/CAs/public.crt + # This job needs both the public CAs to download the datasets over + # HTTPS, and the secret-operator CA to verify Garage. + cat /etc/ssl/certs/ca-certificates.crt /stackable/tls/ca.crt > /tmp/ca-bundle.crt - mc alias set minio https://minio.{{ NAMESPACE }}.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) - cd /tmp + BASE=https://repo.stackable.tech/repository/misc - curl -sO https://repo.stackable.tech/repository/misc/datasets/open-postcode-geo/open-postcode-geo.csv - mc cp open-postcode-geo.csv minio/staging/house-sales/postcode-geo-lookup/ - rm open-postcode-geo.csv + load() { + rclone copyurl "$1" "$2" + } - for year in {2005..2021}; do - curl -sO https://repo.stackable.tech/repository/misc/datasets/uk-house-sales/uk-house-sales-$year.csv - mc cp uk-house-sales-$year.csv minio/staging/house-sales/house-sales/ - rm uk-house-sales-$year.csv + # Please try to order the load jobs from small to large datasets + load "$BASE/datasets/open-postcode-geo/open-postcode-geo.csv" \ + :s3:staging/house-sales/postcode-geo-lookup/open-postcode-geo.csv + + for year in $(seq 2005 2021); do + file="uk-house-sales-$year.csv" + load "$BASE/datasets/uk-house-sales/$file" \ + ":s3:staging/house-sales/house-sales/$file" done - curl -sO https://repo.stackable.tech/repository/misc/earthquake-data/earthquakes_1950_to_2022.csv - mc cp earthquakes_1950_to_2022.csv minio/staging/earthquakes/earthquakes/ - rm earthquakes_1950_to_2022.csv + load "$BASE/earthquake-data/earthquakes_1950_to_2022.csv" \ + :s3:staging/earthquakes/earthquakes/earthquakes_1950_to_2022.csv - curl -sO https://repo.stackable.tech/repository/misc/datasets/e-charging-stations/e-charging-stations-2022-08.csv - mc cp e-charging-stations-2022-08.csv minio/staging/smart-city/e-charging-stations/ - rm e-charging-stations-2022-08.csv + load "$BASE/datasets/e-charging-stations/e-charging-stations-2022-08.csv" \ + :s3:staging/smart-city/e-charging-stations/e-charging-stations-2022-08.csv - curl -sO https://repo.stackable.tech/repository/misc/ny-taxi-data/taxi_zone_lookup.csv - mc cp taxi_zone_lookup.csv minio/staging/taxi/taxi-zone-lookup/ - rm taxi_zone_lookup.csv + load "$BASE/ny-taxi-data/taxi_zone_lookup.csv" \ + :s3:staging/taxi/taxi-zone-lookup/taxi_zone_lookup.csv - curl -sO https://repo.stackable.tech/repository/misc/ny-taxi-data/rate_code_lookup.csv - mc cp rate_code_lookup.csv minio/staging/taxi/rate-code-lookup/ - rm rate_code_lookup.csv + load "$BASE/ny-taxi-data/rate_code_lookup.csv" \ + :s3:staging/taxi/rate-code-lookup/rate_code_lookup.csv - curl -sO https://repo.stackable.tech/repository/misc/ny-taxi-data/payment_type_lookup.csv - mc cp payment_type_lookup.csv minio/staging/taxi/payment-type-lookup/ - rm payment_type_lookup.csv + load "$BASE/ny-taxi-data/payment_type_lookup.csv" \ + :s3:staging/taxi/payment-type-lookup/payment_type_lookup.csv - for month in 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do - curl -sO https://repo.stackable.tech/repository/misc/ny-taxi-data/green_tripdata_$month.parquet - mc cp green_tripdata_$month.parquet minio/staging/taxi/green-tripdata/ - rm green_tripdata_$month.parquet + for month in 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 \ + 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 \ + 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do + file="green_tripdata_$month.parquet" + load "$BASE/ny-taxi-data/$file" ":s3:staging/taxi/green-tripdata/$file" done - for month in 2015-01 2015-02 2015-03 2015-04 2015-05 2015-06 2015-07 2015-08 2015-09 2015-10 2015-11 2015-12 2016-01 2016-02 2016-03 2016-04 2016-05 2016-06 2016-07 2016-08 2016-09 2016-10 2016-11 2016-12 2017-01 2017-02 2017-03 2017-04 2017-05 2017-06 2017-07 2017-08 2017-09 2017-10 2017-11 2017-12 2018-01 2018-02 2018-03 2018-04 2018-05 2018-06 2018-07 2018-08 2018-09 2018-10 2018-11 2018-12 2019-01 2019-02 2019-03 2019-04 2019-05 2019-06 2019-07 2019-08 2019-09 2019-10 2019-11 2019-12 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06 2022-07 2022-08 2022-09 2022-10 2022-11 2022-12 2023-01 2023-02 2023-03 2023-04; do - curl -sO https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet - mc cp yellow_tripdata_$month.parquet minio/staging/taxi/yellow-tripdata/ - rm yellow_tripdata_$month.parquet + for month in 2015-01 2015-02 2015-03 2015-04 2015-05 2015-06 \ + 2015-07 2015-08 2015-09 2015-10 2015-11 2015-12 \ + 2016-01 2016-02 2016-03 2016-04 2016-05 2016-06 \ + 2016-07 2016-08 2016-09 2016-10 2016-11 2016-12 \ + 2017-01 2017-02 2017-03 2017-04 2017-05 2017-06 \ + 2017-07 2017-08 2017-09 2017-10 2017-11 2017-12 \ + 2018-01 2018-02 2018-03 2018-04 2018-05 2018-06 \ + 2018-07 2018-08 2018-09 2018-10 2018-11 2018-12 \ + 2019-01 2019-02 2019-03 2019-04 2019-05 2019-06 \ + 2019-07 2019-08 2019-09 2019-10 2019-11 2019-12 \ + 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 \ + 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 \ + 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 \ + 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 \ + 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06 \ + 2022-07 2022-08 2022-09 2022-10 2022-11 2022-12 \ + 2023-01 2023-02 2023-03 2023-04; do + file="yellow_tripdata_$month.parquet" + load "$BASE/ny-taxi-data/$file" ":s3:staging/taxi/yellow-tripdata/$file" done - for month in 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do - curl -sO https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet - mc cp fhvhv_tripdata_$month.parquet minio/staging/taxi/fhvhv-tripdata/ - rm fhvhv_tripdata_$month.parquet + for month in 2020-09 2020-10 2020-11 2020-12 \ + 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 \ + 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 \ + 2022-01 2022-02 2022-03 2022-04 2022-05 2022-06; do + file="fhvhv_tripdata_$month.parquet" + load "$BASE/ny-taxi-data/$file" ":s3:staging/taxi/fhvhv-tripdata/$file" done + env: + # No config file, everything is configured through the environment. + - name: RCLONE_CONFIG + value: /dev/null + - name: RCLONE_S3_PROVIDER + value: Other + - name: RCLONE_S3_ENDPOINT + value: https://garage.{{ NAMESPACE }}.svc.cluster.local:9000 + # Garage rejects requests signed for a different region. + - name: RCLONE_S3_REGION + value: region-1 + # Garage only serves path style, not virtual-hosted style. + - name: RCLONE_S3_FORCE_PATH_STYLE + value: "true" + - name: RCLONE_CA_CERT + value: /tmp/ca-bundle.crt + # Buckets are created by the Garage init job. Without this rclone + # creates a missing bucket on the fly, which turns a typo into a + # silent success instead of an error. + - name: RCLONE_S3_NO_CHECK_BUCKET + value: "true" + - name: RCLONE_S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + - name: RCLONE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: secretKey volumeMounts: - - name: minio-s3-credentials - mountPath: /minio-s3-credentials - # Mount the certificate generated by the secret-operator - name: tls - mountPath: /etc/minio/mc/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /.mc/certs/CAs - name: certs + mountPath: /stackable/tls volumes: - - name: minio-s3-credentials - secret: - secretName: minio-s3-credentials - # Request a TLS certificate from the secret-operator + # Request the CA cert from the secret-operator to verify the certificate + # served in front of Garage - name: tls ephemeral: volumeClaimTemplate: metadata: annotations: secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio,pod + secrets.stackable.tech/provision-parts: public spec: storageClassName: secrets.stackable.tech accessModes: - ReadWriteOnce resources: requests: + # The secret-operator projects certificates rather than + # provisioning a disk, so it ignores this. The API server + # still requires the field and rejects a non-positive value, + # hence the placeholder. storage: "1" - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory restartPolicy: OnFailure backoffLimit: 50 diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 8bc03804..4d56eed1 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -104,7 +104,7 @@ demos: - kafka - druid - superset - - minio + - garage - s3 - earthquakes manifests: @@ -126,7 +126,7 @@ demos: - kafka - druid - superset - - minio + - garage - s3 - water-levels manifests: @@ -146,7 +146,7 @@ demos: labels: - trino - superset - - minio + - garage - s3 - ny-taxi-data manifests: @@ -166,14 +166,14 @@ demos: labels: - trino - iceberg - - minio + - garage - s3 manifests: [] supportedNamespaces: [] resourceRequests: cpu: 6000m # Measured 5600m memory: 21Gi - pvc: 110Gi # 100Gi for MinIO + pvc: 60Gi # 50Gi for Garage trino-taxi-data: description: Demo loading 2.5 years of New York taxi data into S3 bucket, creating a Trino table and a Superset dashboard documentation: https://docs.stackable.tech/home/stable/demos/trino-taxi-data/ @@ -181,7 +181,7 @@ demos: labels: - trino - superset - - minio + - garage - s3 - ny-taxi-data manifests: @@ -204,7 +204,7 @@ demos: - superset - kafka - nifi - - minio + - garage - s3 - ny-taxi-data - water-levels @@ -220,7 +220,8 @@ demos: resourceRequests: cpu: "80" memory: 200Gi - pvc: 300Gi + # 250Gi Kafka (5 brokers x 50Gi), 50Gi Garage, 34Gi NiFi, 24Gi PostgreSQL + pvc: 400Gi parameters: [] jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data: description: Jupyterhub with PySpark and HDFS integration diff --git a/demos/jupyterhub-keycloak/load-gas-data.yaml b/demos/jupyterhub-keycloak/load-gas-data.yaml index 39f6b7c2..c10d0175 100644 --- a/demos/jupyterhub-keycloak/load-gas-data.yaml +++ b/demos/jupyterhub-keycloak/load-gas-data.yaml @@ -8,14 +8,68 @@ spec: spec: containers: - name: load-gas-data - image: "bitnamilegacy/minio:2024-debian-12" - command: ["bash", "-c", "cd /tmp; curl -O https://repo.stackable.tech/repository/misc/datasets/gas-sensor-data/20160930_203718.csv && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp 20160930_203718.csv minio/demo/gas-sensor/raw/;"] + image: docker.io/rclone/rclone:1.75.1 + command: + - /bin/sh + - -euo + - pipefail + - -c + - | + # This job needs both the public CAs to download the datasets over + # HTTPS, and the secret-operator CA to verify Garage. + cat /etc/ssl/certs/ca-certificates.crt /stackable/tls/ca.crt > /tmp/ca-bundle.crt + + rclone copyurl \ + https://repo.stackable.tech/repository/misc/datasets/gas-sensor-data/20160930_203718.csv \ + :s3:demo/gas-sensor/raw/20160930_203718.csv + env: + # No config file, everything is configured through the environment. + - name: RCLONE_CONFIG + value: /dev/null + - name: RCLONE_S3_PROVIDER + value: Other + - name: RCLONE_S3_ENDPOINT + value: https://garage.{{ NAMESPACE }}.svc.cluster.local:9000 + # Garage rejects requests signed for a different region. + - name: RCLONE_S3_REGION + value: region-1 + # Garage only serves path style, not virtual-hosted style. + - name: RCLONE_S3_FORCE_PATH_STYLE + value: "true" + - name: RCLONE_CA_CERT + value: /tmp/ca-bundle.crt + # Buckets are created by the Garage init job. Without this rclone + # creates a missing bucket on the fly, which turns a typo into a + # silent success instead of an error. + - name: RCLONE_S3_NO_CHECK_BUCKET + value: "true" + - name: RCLONE_S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + - name: RCLONE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: secretKey volumeMounts: - - name: minio-s3-credentials - mountPath: /minio-s3-credentials + - name: tls + mountPath: /stackable/tls volumes: - - name: minio-s3-credentials - secret: - secretName: minio-s3-credentials + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/provision-parts: public + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" restartPolicy: OnFailure backoffLimit: 50 diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml index 6838e500..ba7da00f 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml @@ -61,7 +61,7 @@ data: requirements: - scikit-learn==1.4.0 s3connection: - reference: minio + reference: garage volumes: - name: cm-spark configMap: diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml index fe0bade1..e18a9caf 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml @@ -8,36 +8,67 @@ spec: spec: containers: - name: load-ny-taxi-data - image: "bitnamilegacy/minio:2024-debian-12" - # yamllint disable-line rule:line-length + image: docker.io/rclone/rclone:1.75.1 command: - - "bash" - - "-ce" + - /bin/sh + - -euo + - pipefail + - -c - | - cd /tmp + # This job needs both the public CAs to download the datasets over + # HTTPS, and the secret-operator CA to verify Garage. + cat /etc/ssl/certs/ca-certificates.crt /stackable/tls/ca.crt > /tmp/ca-bundle.crt + + BASE=https://repo.stackable.tech/repository/misc/ny-taxi-data + + # The "prediction" bucket this demo writes its results to is + # created by the Garage init job, along with "demo". for month in 2020-09 2020-10 2020-11 2020-12; do - curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet - mc alias set minio https://minio.{{ NAMESPACE }}.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) - mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ - mc mb --ignore-existing minio/prediction; + file="fhvhv_tripdata_$month.parquet" + # copyurl streams the download straight into S3, no temp file + rclone copyurl "$BASE/$file" ":s3:demo/ny-taxi-data/raw/$file" done + env: + - name: RCLONE_CONFIG + value: /dev/null + - name: RCLONE_S3_PROVIDER + value: Other + - name: RCLONE_S3_ENDPOINT + value: https://garage.{{ NAMESPACE }}.svc.cluster.local:9000 + # Garage rejects requests signed for a different region. + - name: RCLONE_S3_REGION + value: region-1 + # Garage only serves path style, not virtual-hosted style. + - name: RCLONE_S3_FORCE_PATH_STYLE + value: "true" + - name: RCLONE_CA_CERT + value: /tmp/ca-bundle.crt + # Buckets are created by the Garage init job. Without this rclone + # creates a missing bucket on the fly, which turns a typo into a + # silent success instead of an error. + - name: RCLONE_S3_NO_CHECK_BUCKET + value: "true" + - name: RCLONE_S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + - name: RCLONE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: secretKey volumeMounts: - - name: minio-s3-credentials - mountPath: /minio-s3-credentials - name: tls - mountPath: /.mc/certs/CAs + mountPath: /stackable/tls volumes: - - name: minio-s3-credentials - secret: - secretName: minio-s3-credentials - name: tls ephemeral: volumeClaimTemplate: metadata: annotations: secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio + secrets.stackable.tech/provision-parts: public spec: storageClassName: secrets.stackable.tech accessModes: diff --git a/demos/trino-taxi-data/load-test-data.yaml b/demos/trino-taxi-data/load-test-data.yaml index 1eb1c8f8..5212fba5 100644 --- a/demos/trino-taxi-data/load-test-data.yaml +++ b/demos/trino-taxi-data/load-test-data.yaml @@ -8,62 +8,75 @@ spec: spec: containers: - name: load-ny-taxi-data - image: "bitnamilegacy/minio:2024-debian-12" - # yamllint disable-line rule:line-length + image: docker.io/rclone/rclone:1.75.1 command: - - bash - - -ce + - /bin/sh + - -euo + - pipefail + - -c - | - # Copy the CA cert from the "tls" SecretClass - cp -v /etc/minio/mc/original_certs/ca.crt /.mc/certs/CAs/public.crt + # This job needs both the public CAs to download the datasets over + # HTTPS, and the secret-operator CA to verify Garage. + cat /etc/ssl/certs/ca-certificates.crt /stackable/tls/ca.crt > /tmp/ca-bundle.crt - MINIO_ENDPOINT="https://minio.{{ NAMESPACE }}.svc.cluster.local:9000/" - MINIO_ACCESS_KEY=$(cat /minio-s3-credentials/accessKey) - MINIO_SECRET_KEY=$(cat /minio-s3-credentials/secretKey) + BASE=https://repo.stackable.tech/repository/misc/ny-taxi-data - cd /tmp for month in \ - 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 \ - 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 \ - 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do - curl -O "https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet" - mc alias set minio "$MINIO_ENDPOINT" "$MINIO_ACCESS_KEY" "$MINIO_SECRET_KEY" - mc cp "yellow_tripdata_$month.parquet" minio/demo/ny-taxi-data/raw/ + 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 \ + 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 \ + 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do + file="yellow_tripdata_$month.parquet" + # copyurl streams the download straight into S3, no temp file + rclone copyurl "$BASE/$file" ":s3:demo/ny-taxi-data/raw/$file" done + env: + - name: RCLONE_CONFIG + value: /dev/null + - name: RCLONE_S3_PROVIDER + value: Other + - name: RCLONE_S3_ENDPOINT + value: https://garage.{{ NAMESPACE }}.svc.cluster.local:9000 + - name: RCLONE_S3_REGION + value: region-1 + # Garage only serves path style, not virtual-hosted style. + - name: RCLONE_S3_FORCE_PATH_STYLE + value: "true" + - name: RCLONE_CA_CERT + value: /tmp/ca-bundle.crt + # Buckets are created by the Garage init job. Without this rclone + # creates a missing bucket on the fly, which turns a typo into a + # silent success instead of an error. + - name: RCLONE_S3_NO_CHECK_BUCKET + value: "true" + - name: RCLONE_S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + - name: RCLONE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: secretKey volumeMounts: - - name: minio-s3-credentials - mountPath: /minio-s3-credentials - # Mount the certificate generated by the secret-operator - name: tls - mountPath: /etc/minio/mc/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /.mc/certs/CAs - name: certs - + mountPath: /stackable/tls volumes: - - name: minio-s3-credentials - secret: - secretName: minio-s3-credentials - # Request a TLS certificate from the secret-operator - name: tls ephemeral: volumeClaimTemplate: metadata: annotations: secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio + # Only the CA cert is needed, to verify the certificate served + # in front of Garage. No keypair is issued, so no scope either. + secrets.stackable.tech/provision-parts: public spec: storageClassName: secrets.stackable.tech accessModes: - ReadWriteOnce resources: requests: - storage: 1 - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory + storage: "1" restartPolicy: OnFailure backoffLimit: 50 diff --git a/docs/modules/demos/images/airflow-scheduled-job/overview.drawio.png b/docs/modules/demos/images/airflow-scheduled-job/overview.drawio.png new file mode 100644 index 00000000..4db831f0 Binary files /dev/null and b/docs/modules/demos/images/airflow-scheduled-job/overview.drawio.png differ diff --git a/docs/modules/demos/images/airflow-scheduled-job/overview.png b/docs/modules/demos/images/airflow-scheduled-job/overview.png deleted file mode 100644 index 4d859d59..00000000 Binary files a/docs/modules/demos/images/airflow-scheduled-job/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/argo-cd-git-ops/architecture-overview.drawio.svg b/docs/modules/demos/images/argo-cd-git-ops/architecture-overview.drawio.svg index 0a4ca03f..05bbbe98 100644 --- a/docs/modules/demos/images/argo-cd-git-ops/architecture-overview.drawio.svg +++ b/docs/modules/demos/images/argo-cd-git-ops/architecture-overview.drawio.svg @@ -1,4 +1,1428 @@ - - - -
Bootstrapping
ArgoCD via
Helm Chart
Apply ArgoCD
Projects
Apply ArgoCD
Applications
Apply ArgoCD
ApplicationSets

stackablectl
Projects
- stackable-operators
- minio
- airflow
ApplicationSets
- stackable-operators
Airflow manifests,
secrets, credentials
Install via
Helm Chart
Helm / Container registry
Apply operator 
Helm charts
from registry
Apply 
sealed-secrets-controller,
airflow-postgres,
via Helm chart
Apply
airflow
manifets
from Git
Apply manifests,
credentials, clusterroles
Update Airflow
manifest, e.g. 
version update,
increase replicas
Cluster
Admin
DAG
Developer
create

airflow-operator
read by
Airflow
Cluster
Update ArgoCD
projects and
applications e.g.,
stackable release
version
             sealed-
                 secrets-
                 controller
Airflow
data
read/write
logs
airflow-credentials
read and decrypted by
sealed-airflow-credentials
create
decrypted
secret
read by
Airflow DAGs
Synchronize DAGs
via gitsync
Update DAGs
(can be a different
Git repository)
Applications
- sealed-secrets-controller
- minio
- airflow
- airflow-postgres
 \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ Bootstrapping +
+
+ ArgoCD via +
+
+ Helm Chart +
+
+
+
+ + + + Bootstrapping + + + ArgoCD via + + + Helm Chart + + + + + + + + + + + + + + + + + +
+
+
+
+ Apply ArgoCD +
+
+ Projects +
+
+
+
+ + + + Apply ArgoCD + + + Projects + + + + + + + + + + + + + + + + + +
+
+
+
+ Apply ArgoCD +
+
+ Applications +
+
+
+
+ + + + Apply ArgoCD + + + Applications + + + + + + + + + + + + + + + + + +
+
+
+
+ Apply ArgoCD +
+
+ ApplicationSets +
+
+
+
+ + + + Apply ArgoCD + + + ApplicationSets + + + + + + + + + + + + + + + +
+
+
+
+
+ + stackablectl + +
+
+
+
+ + + + + stackablectl + + + + + + + + + + + + + + + + +
+
+
+ Projects +
+
+
+ + + Projects + + + + + + + + + + + + +
+
+
+ - stackable-operators +
+ - minio +
+
+ - airflow +
+
+
+
+ + + + - stackable-operators + + + - minio + + + - airflow + + + + + + + + + + + + + + + + + +
+
+
+ ApplicationSets +
+
+
+ + + ApplicationSets + + + + + + + + + + + + +
+
+
+ - stackable-operators +
+
+
+ + + - stackable-operators + + + + + + + + + + + + + + + + + + + +
+
+
+
+ + Airflow manifests, + +
+
+ + secrets, credentials + +
+
+
+
+ + + + Airflow manifests, + + + secrets, credentials + + + + + + + + + + + + + + + + +
+
+
+
+ Install via +
+
+ Helm Chart +
+
+
+
+ + + + Install via + + + Helm Chart + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ Helm / Container registry +
+
+
+ + + Helm / Container registry + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ Apply operator +
+
+ Helm charts +
+
+ from registry +
+
+
+
+ + + + Apply operator + + + Helm charts + + + from registry + + + + + + + + + + + + + + + + + +
+
+
+
+ Apply +
+
+ sealed-secrets-controller, +
+
+ airflow-postgres, +
+
+ via Helm chart +
+
+
+
+ + + + Apply + + + sealed-secrets-controller, + + + airflow-postgres, + + + via Helm chart + + + + + + + + + + + + + + + + + + +
+
+
+
+ Apply +
+
+ airflow +
+
+ manifets +
+
+ from Git +
+
+
+
+ + + + Apply + + + airflow + + + manifets + + + from Git + + + + + + + + + + + + + + + + +
+
+
+
+ Apply manifests, +
+
+ credentials, clusterroles +
+
+
+
+ + + + Apply manifests, + + + credentials, clusterroles + + + + + + + + + + + + + + + + + +
+
+
+ Update Airflow +
+ manifest, e.g. +
+ version update, +
+ increase replicas +
+
+
+ + + + Update Airflow + + + manifest, e.g. + + + version update, + + + increase replicas + + + + + + + + + + + + + + + + +
+
+
+
+ Cluster +
+
+ Admin +
+
+
+
+ + + + Cluster + + + Admin + + + + + + + + + + + + + + + +
+
+
+
+ DAG +
+
+ Developer +
+
+
+
+ + + + DAG + + + Developer + + + + + + + + + + + + + + + + +
+
+
+ create +
+
+
+ + + create + + + + + + + + + + + + + + +
+
+
+
+
+
+
+ + airflow-operator + +
+
+
+
+ + + + + airflow-operator + + + + + + + + + + + + + + + + +
+
+
+ read by +
+
+
+ + + read by + + + + + + + + + + + + + + + +
+
+
+
+ + Airflow + +
+
+ + Cluster + +
+
+
+
+ + + + Airflow + + + Cluster + + + + + + + + + + + + + + +
+
+
+ S3 +
+
+
+ + + S3 + + + + + + + + + + + + + + + +
+
+
+ Update ArgoCD +
+ projects and +
+ applications e.g., +
+ stackable release +
+ version +
+
+
+ + + + Update ArgoCD + + + projects and + + + applications e.g., + + + stackable release + + + version + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ + + + sealed- + +
+
+ + secrets- + +
+
+ + controller + +
+
+
+
+ + + + sealed- + + + secrets- + + + controller + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ Airflow +
+
+ data +
+
+
+
+ + + + Airflow + + + data + + + + + + + + + + + + + + + + + +
+
+
+
+ read/write +
+
+ logs +
+
+
+
+ + + + read/write + + + logs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ airflow-credentials +
+
+
+ + + airflow-credentials + + + + + + + + + + + + + + + +
+
+
+ read and decrypted by +
+
+
+ + + read and decrypted by + + + + + + + + + + + + + + + +
+
+
+ sealed-airflow-credentials +
+
+
+ + + sealed-airflow-credentials + + + + + + + + + + + + + + + +
+
+
+
+ create +
+
+ decrypted +
+
+ secret +
+
+
+
+ + + + create + + + decrypted + + + secret + + + + + + + + + + + + + + + + + +
+
+
+ read by +
+
+
+ + + read by + + + + + + + + + + + + + + +
+
+
+ + Airflow DAGs + +
+
+
+ + + Airflow DAGs + + + + + + + + + + + + + + + +
+
+
+
+ Synchronize DAGs +
+
+ via gitsync +
+
+
+
+ + + + Synchronize DAGs + + + via gitsync + + + + + + + + + + + + + + + + + +
+
+
+ Update DAGs +
+ (can be a different +
+ Git repository) +
+
+
+ + + + Update DAGs + + + (can be a different + + + Git repository) + + + + + + + + + + + + + + + + + + + +
+
+
+ Applications +
+
+
+ + + Applications + + + + + + + + + + + + + +
+
+
+ - sealed-secrets-controller +
+ - minio +
+
+ - airflow +
+
+ - airflow-postgres +
+
+
+
+ + + + - sealed-secrets-controller + + + - minio + + + - airflow + + + - airflow-postgres + + + + + + + + + + + + + + + + + + + + + + + + + Text is not SVG - cannot display + + + + diff --git a/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/overview.drawio.png b/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/overview.drawio.png new file mode 100644 index 00000000..4b450e27 Binary files /dev/null and b/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/overview.drawio.png differ diff --git a/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/overview.png b/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/overview.png deleted file mode 100644 index 3d0d878c..00000000 Binary files a/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/hbase-hdfs-load-cycling-data/overview.drawio.png b/docs/modules/demos/images/hbase-hdfs-load-cycling-data/overview.drawio.png new file mode 100644 index 00000000..0f5d994c Binary files /dev/null and b/docs/modules/demos/images/hbase-hdfs-load-cycling-data/overview.drawio.png differ diff --git a/docs/modules/demos/images/hbase-hdfs-load-cycling-data/overview.png b/docs/modules/demos/images/hbase-hdfs-load-cycling-data/overview.png deleted file mode 100644 index d9f79e4f..00000000 Binary files a/docs/modules/demos/images/hbase-hdfs-load-cycling-data/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/logging/overview.drawio.png b/docs/modules/demos/images/logging/overview.drawio.png new file mode 100644 index 00000000..2896707b Binary files /dev/null and b/docs/modules/demos/images/logging/overview.drawio.png differ diff --git a/docs/modules/demos/images/logging/overview.png b/docs/modules/demos/images/logging/overview.png deleted file mode 100644 index f2289f11..00000000 Binary files a/docs/modules/demos/images/logging/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/overview.drawio.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/overview.drawio.png new file mode 100644 index 00000000..1c69db4b Binary files /dev/null and b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/overview.drawio.png differ diff --git a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/overview.png b/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/overview.png deleted file mode 100644 index 9ef04dc1..00000000 Binary files a/docs/modules/demos/images/nifi-kafka-druid-earthquake-data/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/overview.drawio.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/overview.drawio.png new file mode 100644 index 00000000..20730015 Binary files /dev/null and b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/overview.drawio.png differ diff --git a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/overview.png b/docs/modules/demos/images/nifi-kafka-druid-water-level-data/overview.png deleted file mode 100644 index eae66ee2..00000000 Binary files a/docs/modules/demos/images/nifi-kafka-druid-water-level-data/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/signal-processing/overview.drawio.png b/docs/modules/demos/images/signal-processing/overview.drawio.png new file mode 100644 index 00000000..a3130a9a Binary files /dev/null and b/docs/modules/demos/images/signal-processing/overview.drawio.png differ diff --git a/docs/modules/demos/images/signal-processing/overview.png b/docs/modules/demos/images/signal-processing/overview.png deleted file mode 100644 index f23a86a2..00000000 Binary files a/docs/modules/demos/images/signal-processing/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/spark-k8s-anomaly-detection-taxi-data/overview.drawio.png b/docs/modules/demos/images/spark-k8s-anomaly-detection-taxi-data/overview.drawio.png new file mode 100644 index 00000000..e87b83c5 Binary files /dev/null and b/docs/modules/demos/images/spark-k8s-anomaly-detection-taxi-data/overview.drawio.png differ diff --git a/docs/modules/demos/images/spark-k8s-anomaly-detection-taxi-data/overview.png b/docs/modules/demos/images/spark-k8s-anomaly-detection-taxi-data/overview.png deleted file mode 100644 index 774ba519..00000000 Binary files a/docs/modules/demos/images/spark-k8s-anomaly-detection-taxi-data/overview.png and /dev/null differ diff --git a/docs/modules/demos/images/trino-taxi-data/overview.drawio.png b/docs/modules/demos/images/trino-taxi-data/overview.drawio.png new file mode 100644 index 00000000..45ff66a5 Binary files /dev/null and b/docs/modules/demos/images/trino-taxi-data/overview.drawio.png differ diff --git a/docs/modules/demos/images/trino-taxi-data/overview.png b/docs/modules/demos/images/trino-taxi-data/overview.png deleted file mode 100644 index c6fbcb3c..00000000 Binary files a/docs/modules/demos/images/trino-taxi-data/overview.png and /dev/null differ diff --git a/docs/modules/demos/pages/airflow-scheduled-job.adoc b/docs/modules/demos/pages/airflow-scheduled-job.adoc index f7b807ee..2968568c 100644 --- a/docs/modules/demos/pages/airflow-scheduled-job.adoc +++ b/docs/modules/demos/pages/airflow-scheduled-job.adoc @@ -40,7 +40,7 @@ This demo will ** *Open Policy Agent*: An open-source policy engine used for user authorization ** *Trino*: A fast distributed SQL query engine for big data analytics that helps you explore your data universe. This demo uses it to enable SQL access to the data -** *MinIO*: A S3 compatible object store. This demo uses it as persistent storage to store the Trino data and Airflow logs +** *Garage*: S3 compatible object store. This demo uses it as persistent storage to store the Trino data and Airflow logs * Mount several Airflow jobs (referred to as Directed Acyclic Graphs, or DAGs) for the cluster to use * Enable and schedule the jobs * Verify the job status with the Airflow Webserver UI @@ -49,7 +49,7 @@ This demo will You can see the deployed products and their relationship in the following diagram: -image::airflow-scheduled-job/overview.png[] +image::airflow-scheduled-job/overview.drawio.png[] == List deployed Stackable services @@ -75,8 +75,6 @@ $ stackablectl stacklet list -n airflow-demo │ opa ┆ opa ┆ default ┆ ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ trino ┆ trino ┆ default ┆ coordinator-https https://172.19.0.5:31087 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ minio ┆ minio-console ┆ default ┆ https https://172.19.0.4:31792 ┆ │ └─────────┴───────────────┴───────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- @@ -133,8 +131,8 @@ Click on the `run_every_minute` box in the centre of the page to select the logs [WARNING] ==== In this demo, the KubernetesExecutor is deployed which means that logs are only preserved (and available in the UI) if either remote logging or the SDP logging framework is configured. -In this demo we set up remote logging using S3/Minio. -Since Minio in this case is set up with TLS, the Airflow connection requires that the webserver has access to a relevant certificate and that every pod has environment variables containing the access and secret keys. +In this demo we set up remote logging to S3. +Since S3 is served over TLS here, the Airflow connection requires that the webserver has access to a relevant certificate and that every pod has environment variables containing the access and secret keys. See the https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/stable/kubernetes_executor.html#managing-dags-and-logs[Airflow Documentation{external-link-icon}^] for more details. If you are interested in persisting the logs using the SDP logging framework, take a look at the xref:logging.adoc[] demo. diff --git a/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc b/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc index 8c9be204..694e896f 100644 --- a/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc +++ b/docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc @@ -35,7 +35,7 @@ Instance types that loosely correspond to this on the Hyperscalers are: * *Azure*: `Standard_D4_v2` * *AWS*: `m5.2xlarge` -In addition to these nodes the operators will request multiple persistent volumes with a total capacity of about 300Gi. +In addition to these nodes the operators will request multiple persistent volumes with a total capacity of about 400Gi. == Overview @@ -47,7 +47,7 @@ This demo will This demo uses it to enable SQL access to the data. ** *Apache Spark*: A multi-language engine for executing data engineering, data science, and machine learning. This demo uses it to stream data from Kafka into the lakehouse. -** *MinIO*: S3 compatible object store. +** *Garage*: S3 compatible object store. This demo uses it as persistent storage to store all the data used ** *Apache Kafka*: A distributed event streaming platform for high-performance data pipelines, streaming analytics and data integration. This demo uses it as an event streaming platform to stream the data in near real-time. @@ -68,7 +68,7 @@ This demo will You can see the deployed products and their relationship in the following diagram: -image::data-lakehouse-iceberg-trino-spark/overview.png[] +image::data-lakehouse-iceberg-trino-spark/overview.drawio.png[] == Apache Iceberg @@ -129,28 +129,25 @@ $ stackablectl stacklet list │ ┆ ┆ ┆ coordinator-https https://212.227.194.245:30841 ┆ │ ├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ zookeeper ┆ zookeeper ┆ default ┆ ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ minio ┆ minio-console ┆ default ┆ http http://217.160.99.235:30238 ┆ │ └───────────┴───────────────┴───────────┴────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- include::partial$instance-hint.adoc[] -== MinIO +== S3 === Listing Buckets -The S3 provided by MinIO is used as persistent storage to store all the data used. Open the `minio` endpoint -`http` retrieved by the `stackablectl stacklet list` command in your browser (http://217.160.99.235:30238 in -this case). - -image::data-lakehouse-iceberg-trino-spark/minio_1.png[] +The S3 provided by Garage is used as persistent storage to store all the data used. -Log in with the username `admin` and password `adminadmin`. +include::partial$inspect-s3.adoc[] -image::data-lakehouse-iceberg-trino-spark/minio_2.png[] +[source,console] +---- +$ rclone lsd :s3: +---- -Here, you can see the two buckets contained in the S3: +There are two buckets: . `staging`: The demo loads static datasets into this area. It is stored in different formats, such as CSV and Parquet. It does contain actual data tables as well as lookup tables. @@ -159,21 +156,28 @@ Here, you can see the two buckets contained in the S3: === Inspecting Lakehouse -Click on the bucket `lakehouse`. +Each prefix in `lakehouse` holds a different dataset: -image::data-lakehouse-iceberg-trino-spark/minio_3.png[] - -Multiple folders (called prefixes in S3), each containing a different dataset, are displayed. First, select the folder -`house-sales`, then the folder starting with `house-sales-*`, and lastly, the folder named `data`. +[source,console] +---- +$ rclone lsf :s3:lakehouse +---- -image::data-lakehouse-iceberg-trino-spark/minio_4.png[] +Look at the `house-sales` dataset. +Each table lives in a directory named after it followed by a random suffix, such as `house-sales-c381a26d1b8f4f52babc7481ef1dd815`, so list the prefix rather than guessing the name. +Below that directory, the `data` prefix is partitioned by year: -As you can see, the table `house-sales` is partitioned by year. Go ahead and click on any folder. +[source,console] +---- +$ rclone lsf --dirs-only --recursive :s3:lakehouse/house-sales +---- -image::data-lakehouse-iceberg-trino-spark/minio_5.png[] +Listing one of those partitions shows the files Trino wrote, containing all the house sales for that particular year: -You can see that Trino has data into the selected folder containing all the house sales of that -particular year. +[source,console] +---- +$ rclone lsl :s3:lakehouse/house-sales +---- == NiFi diff --git a/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc b/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc index 08726333..996991b7 100644 --- a/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc +++ b/docs/modules/demos/pages/hbase-hdfs-load-cycling-data.adoc @@ -42,7 +42,7 @@ This demo will You can see the deployed products and their relationship in the following diagram: -image::hbase-hdfs-load-cycling-data/overview.png[] +image::hbase-hdfs-load-cycling-data/overview.drawio.png[] == Listing the deployed Stackable services diff --git a/docs/modules/demos/pages/index.adoc b/docs/modules/demos/pages/index.adoc index bbde1231..1070b097 100644 --- a/docs/modules/demos/pages/index.adoc +++ b/docs/modules/demos/pages/index.adoc @@ -22,7 +22,7 @@ Below is an incomplete list of third-party components referenced in these demos: * JupyterHub * JupyterLab * Keycloak -* MinIO +* Garage * Ollama * OpenLDAP * OpenSearch Dashboards diff --git a/docs/modules/demos/pages/jupyterhub-keycloak.adoc b/docs/modules/demos/pages/jupyterhub-keycloak.adoc index 26e31931..c4366104 100644 --- a/docs/modules/demos/pages/jupyterhub-keycloak.adoc +++ b/docs/modules/demos/pages/jupyterhub-keycloak.adoc @@ -41,7 +41,7 @@ This demo highlights several key features: * Secure Authentication: Utilizes {keycloak}[Keycloak] for robust user authentication and identity management * Dynamic Spark Integration: Demonstrates how to start a distributed Spark cluster directly from a Jupyter notebook, with dynamic resource allocation -* S3 Storage Interaction: Illustrates reading from and writing to an S3-compatible storage (MinIO) using Spark, with secure credential management +* S3 Storage Interaction: Illustrates reading from and writing to an S3-compatible storage (Garage) using Spark, with secure credential management * Scalable and Flexible: Leverages Kubernetes for scalable resource management, allowing users to select from predefined resource profiles * User-Friendly: Provides an intuitive interface for data scientists to perform common data operations with ease @@ -51,7 +51,7 @@ This demo will: * Spin up the following data products: ** JupyterHub: A multi-user server for Jupyter notebooks ** Keycloak: An identity and access management product -** S3: A Minio instance for data storage +** S3: A Garage instance for data storage * Download a sample of {gas-sensor}[gas sensor measurements*] into S3 * Install the Jupyter notebook * Demonstrate some basic data operations against S3 @@ -84,8 +84,8 @@ This setup is ideal for interactive data processing. === S3 Storage Integration -* **MinIO**: Utilizes MinIO as an S3-compatible storage solution for storing and retrieving data. -* **Secure Credential Management**: MinIO credentials are managed using Kubernetes secrets, keeping them separate from notebook code. +* **Garage**: Utilizes Garage as an S3-compatible storage solution for storing and retrieving data. +* **Secure Credential Management**: S3 credentials are managed using Kubernetes secrets, keeping them separate from notebook code. * **Data Operations**: Demonstrates reading from and writing to S3 storage using Spark, with support for CSV and Parquet formats. == Configuration Settings Overview @@ -112,9 +112,9 @@ For more details, see the https://docs.stackable.tech/home/stable/tutorials/jupy == Detailed Demo/Notebook Walkthrough The demo showcases an https://github.com/stackabletech/demos/blob/main/stacks/jupyterhub-keycloak/process-s3.ipynb[ipython notebook] that begins by outputting the versions of Python, Java, and PySpark being used. -It reads MinIO credentials from a mounted secret to access the S3 storage. +It reads S3 credentials from a mounted secret to access the S3 storage. This ensures that the environment is correctly set up and that the necessary credentials are available for S3 operations. -The notebook configures Spark to interact with an S3 bucket hosted on MinIO. +The notebook configures Spark to interact with an S3 bucket hosted on Garage. It includes necessary Hadoop and AWS libraries to facilitate S3 operations. The Spark session is configured with various settings, including executor instances, memory, and cores, to ensure optimal performance. @@ -141,7 +141,7 @@ NAME READY STATUS RESTARTS AGE hub-84f49ccbd7-29h7j 1/1 Running 0 56m keycloak-544d757f57-f55kr 2/2 Running 0 57m load-gas-data-m6z5p 0/1 Completed 0 54m -minio-5486d7584f-x2jn8 1/1 Running 0 57m +garage-6b9d4c7f85-2vqnt 2/2 Running 0 57m proxy-648bf7f45b-62vqg 1/1 Running 0 56m ---- @@ -248,20 +248,27 @@ You can find this in the JupyterHub UI at http://:31095/hub/admin e.g http:/ image::jupyterhub-keycloak/admin-tab.png[] -You can inspect the S3 buckets by using stackable stacklet list to return the Minio endpoint and logging in there with `admin/adminadmin`: +You can inspect the S3 buckets with any S3 client. + +include::partial$inspect-s3.adoc[] [source,console] ---- -$ stackablectl stacklet list - -┌─────────┬───────────────┬───────────┬───────────────────────────────┬────────────┐ -│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │ -╞═════════╪═══════════════╪═══════════╪═══════════════════════════════╪════════════╡ -│ minio ┆ minio-console ┆ default ┆ http http://172.19.0.5:32470 ┆ │ -└─────────┴───────────────┴───────────┴───────────────────────────────┴────────────┘ +$ rclone lsl :s3:demo ---- -image::jupyterhub-keycloak/s3-buckets.png[] +[source,text] +---- + 0 2026-10-06 11:55:43.940000000 gas-sensor/agg/_SUCCESS + 18960 2026-10-06 11:55:43.740000000 gas-sensor/agg/part-00000-90d66bed-f580-4948-ba0e-a6982ee9da01-c000.snappy.parquet + 0 2026-10-06 11:55:36.566000000 gas-sensor/parquet/_SUCCESS + 5571983 2026-10-06 11:55:36.424000000 gas-sensor/parquet/part-00000-67fe8c0e-6c1b-438b-b39e-4912f313e59e-c000.snappy.parquet + 4698504 2026-10-06 11:55:36.476000000 gas-sensor/parquet/part-00001-67fe8c0e-6c1b-438b-b39e-4912f313e59e-c000.snappy.parquet + 46497077 2023-04-28 17:35:33.000000000 gas-sensor/raw/20160930_203718.csv + 0 2026-10-06 11:55:31.844000000 gas-sensor/rewritten/_SUCCESS + 25345669 2026-10-06 11:55:31.460000000 gas-sensor/rewritten/part-00000-16062861-51e4-42a1-8698-232c21d5dace-c000.csv + 21151302 2026-10-06 11:55:31.556000000 gas-sensor/rewritten/part-00001-16062861-51e4-42a1-8698-232c21d5dace-c000.csv +---- NOTE: if you attempt to re-run the notebook you will need to first remove the `_temporary folders` from the S3 buckets. These are created by spark jobs and are not removed from the bucket when the job has completed. @@ -270,7 +277,7 @@ These are created by spark jobs and are not removed from the bucket when the job === Add your own data -You can augment the demo dataset with your own data by creating new buckets and folders and uploading your own data via the MinIO UI. +You can augment the demo dataset with your own data by creating new buckets and folders and uploading your own data with an S3 client. === Scale up and out diff --git a/docs/modules/demos/pages/logging.adoc b/docs/modules/demos/pages/logging.adoc index 9b6810bf..e789cbac 100644 --- a/docs/modules/demos/pages/logging.adoc +++ b/docs/modules/demos/pages/logging.adoc @@ -43,7 +43,7 @@ This demo will You can see the deployed products and their relationship in the following diagram: -image::logging/overview.png[] +image::logging/overview.drawio.png[] == List the deployed Stackable services diff --git a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc index 464cd93d..055f4ffd 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc @@ -38,7 +38,7 @@ This demo will from the internet and ingest it into Kafka. ** *Druid*: A real-time database to power modern analytics applications. This demo uses it to ingest the near real-time data from Kafka, store it and enable access to the data via SQL. -** *MinIO*: A S3 compatible object store. This demo uses it as persistent storage for Druid to store all the data. +** *Garage*: S3 compatible object store. This demo uses it as persistent storage for Druid to store all the data. * Continuously emit approximately 10,000 records/s of https://earthquake.usgs.gov/[earthquake data] into Kafka. * Start a Druid ingestion job that ingests the data into the Druid instance. * Create Superset dashboards for visualization of the data. @@ -46,7 +46,7 @@ This demo will The whole data pipeline will have a very low latency, from putting a record into Kafka to showing up in the dashboard charts. You can see the deployed products and their relationship in the following diagram: -image::nifi-kafka-druid-earthquake-data/overview.png[] +image::nifi-kafka-druid-earthquake-data/overview.drawio.png[] == List the deployed Stackable services @@ -72,8 +72,6 @@ $ stackablectl stacklet list │ superset ┆ superset ┆ default ┆ node-http http://172.19.0.4:30769 ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ zookeeper ┆ zookeeper ┆ default ┆ server-zk zookeeper-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ minio ┆ minio-console ┆ default ┆ http http://172.19.0.3:32007 ┆ │ └───────────┴───────────────┴───────────┴─────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- @@ -230,7 +228,7 @@ It uses the `CSVReader` to parse the downloaded CSV and the `JsonRecordSetWriter Druid is used to ingest the near real-time data from Kafka, store it and enable SQL access. The demo has started an ingestion job reading earthquake records from the Kafka topic earthquakes and saving them into Druid's deep storage. -The Druid deep storage is based on the S3 store provided by MinIO. +The Druid deep storage is based on the S3 store provided by Garage. === View the Ingestion job @@ -363,29 +361,30 @@ order by 1 desc image::nifi-kafka-druid-earthquake-data/superset_12.png[] -== MinIO - -The S3 provided by MinIO is used as a persistent deep storage for Druid to store all the data used. -Open the `minio` endpoint `http` in your browser (http://172.19.0.3:32007 in this case). +== S3 -image::nifi-kafka-druid-earthquake-data/minio_1.png[] +The S3 provided by Garage is used as a persistent deep storage for Druid to store all the data used. -Log in with the username `admin` and password `adminadmin`. +include::partial$inspect-s3.adoc[] -image::nifi-kafka-druid-earthquake-data/minio_2.png[] +Druid writes its segments into the `demo` bucket, under `data/earthquakes`: -Click on the bucket `demo` and open the folders `data` -> `earthquakes`. - -image::nifi-kafka-druid-earthquake-data/minio_3.png[] +[source,console] +---- +$ rclone lsf --dirs-only :s3:demo/data/earthquakes +---- -As you can see Druid saved 201.5 MiB of data within 73 prefixes (folders). -One prefix corresponds to one segment which in turn contains all the data of a year. -If you don't see any folders or files, the reason is that Druid has not saved its data from memory to the deep storage yet. -After waiting for roughly an hour, the data should have been flushed to S3 and show up. +Each prefix corresponds to one segment, which in turn contains all the data of a year. +Druid saved roughly 201.5 MiB of data across 73 prefixes. +If nothing is listed, Druid has not flushed its data from memory to the deep storage yet. +After waiting for roughly an hour, the data should show up. -image::nifi-kafka-druid-earthquake-data/minio_4.png[] +Opening up a prefix for a specific year shows the file containing the data of that year: -If you open up a prefix for a specific year you can see that Druid has placed a file containing the data of that year there. +[source,console] +---- +$ rclone lsl :s3:demo/data/earthquakes +---- == Summary diff --git a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc index 1b04f96f..fa8217fa 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc @@ -38,7 +38,7 @@ This demo will from the internet and ingest it into Kafka. ** *Druid*: A real-time database to power modern analytics applications. This demo uses it to ingest the near real-time data from Kafka, store it and enable access to the data via SQL. -** *MinIO*: A S3 compatible object store. This demo uses it as persistent storage for Druid to store all the data. +** *Garage*: S3 compatible object store. This demo uses it as persistent storage for Druid to store all the data. * Ingest water level data from the {pegelonline}[PEGELONLINE web service] into Kafka. The data contains measured water levels of different measuring stations all around Germany. If the web service is unavailable, this demo will not work, as it needs the web service to ingest the data. @@ -51,7 +51,7 @@ This demo will The whole data pipeline will have a very low latency, from putting a record into Kafka to showing up in the dashboard charts. You can see the deployed products and their relationship in the following diagram: -image::nifi-kafka-druid-water-level-data/overview.png[] +image::nifi-kafka-druid-water-level-data/overview.drawio.png[] == List the deployed Stackable services @@ -77,8 +77,6 @@ $ stackablectl stacklet list │ superset ┆ superset ┆ default ┆ node-http http://172.19.0.3:30435 ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ zookeeper ┆ zookeeper ┆ default ┆ server-zk zookeeper-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ minio ┆ minio-console ┆ default ┆ http http://172.19.0.5:30196 ┆ │ └───────────┴───────────────┴───────────┴─────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- @@ -395,7 +393,7 @@ corner. Druid is used to ingest the near real-time data from Kafka, store it and enable SQL access. The demo has started two ingestion jobs - one reading from the topic `stations` and the other from `measurements` - and saving it into Druid's -deep storage. The Druid deep storage is based on the S3 store provided by MinIO. +deep storage. The Druid deep storage is based on the S3 store provided by Garage. === View the Ingestion job @@ -564,32 +562,37 @@ order by 2 desc image::nifi-kafka-druid-water-level-data/superset_13.png[] -== MinIO - -The S3 MinIO store provides persistent deep storage for Druid to store all the data used. Open the MinIO endpoint -`http` retrieved by `stackablectl stacklet list` in your browser (http://172.19.0.5:30196 in this case). +== S3 -image::nifi-kafka-druid-water-level-data/minio_1.png[] - -Log in with the username `admin` and password `adminadmin`. +The S3 provided by Garage is used as persistent deep storage for Druid to store all the data used. -image::nifi-kafka-druid-water-level-data/minio_2.png[] +include::partial$inspect-s3.adoc[] -Click on the bucket `demo` and open the folder `data`. +Druid writes its segments into the `demo` bucket, under `data`, with one directory per data source: -image::nifi-kafka-druid-water-level-data/minio_3.png[] +[source,console] +---- +$ rclone lsf --dirs-only :s3:demo/data +---- -You can see that Druid has created a folder for both data sources. Go ahead and open the folder `measurements`. +Look at `measurements`: -image::nifi-kafka-druid-water-level-data/minio_4.png[] +[source,console] +---- +$ rclone lsf --dirs-only :s3:demo/data/measurements +---- -Druid saved 51.5 MiB of data within 33 prefixes (folders). One prefix corresponds to one segment, which contains all the -measurements of a day. If you don't see any folders or files, the reason is that Druid still needs to save its data from -memory to the deep storage. After waiting for roughly an hour, the data should have been flushed to S3 and show up. +Druid saved roughly 51.5 MiB of data across 33 prefixes. +One prefix corresponds to one segment, which contains all the measurements of a day. +If nothing is listed, Druid still needs to save its data from memory to the deep storage. +After waiting for roughly an hour, the data should show up. -image::nifi-kafka-druid-water-level-data/minio_5.png[] +Opening up a prefix for a specific day shows the file containing that day's data: -If you open up a prefix for a specific day, you can see that Druid has placed a file containing that day's data there. +[source,console] +---- +$ rclone lsl :s3:demo/data/measurements +---- == Summary diff --git a/docs/modules/demos/pages/signal-processing.adoc b/docs/modules/demos/pages/signal-processing.adoc index 21c95c27..30634fe7 100644 --- a/docs/modules/demos/pages/signal-processing.adoc +++ b/docs/modules/demos/pages/signal-processing.adoc @@ -141,7 +141,7 @@ There are two located in the "Stackable Data Platform" folder. The _Gas measurements_ dashboard shows the original data. The first graph plots two measurments (`r1`, `r2`), together with the model scores (`r1_score`, `r2_score`, `r1_score_lttb`).# These are superimposed on each other for ease of comparison. -image::signal-processing/measurements.png[] +image::signal-processing/measurements.drawio.png[] === Predictions diff --git a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc index f1af6eed..733b03c3 100644 --- a/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc +++ b/docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc @@ -1,5 +1,5 @@ = spark-k8s-anomaly-detection-taxi-data -:description: Deploy a Kubernetes-based Spark demo for anomaly detection using the popular New York taxi dataset, featuring Trino, Spark, MinIO, and Superset. +:description: Deploy a Kubernetes-based Spark demo for anomaly detection using the popular New York taxi dataset, featuring Trino, Spark, S3, and Superset. :scikit-lib: https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html :k8s-cpu: https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/#cpu @@ -38,7 +38,7 @@ This demo will it to batch-process data from S3 by training and scoring an unsupervised anomaly detection model and writing the results into a Trino table. Spark uses an isolation forest algorithm from the scikit-learn machine learning library in this demo. -** *MinIO*: A S3 compatible object store. This demo uses it as persistent storage to store all the data used +** *Garage*: S3 compatible object store. This demo uses it as persistent storage to store all the data used ** *Hive metastore*: A service that stores metadata related to Apache Hive and other services. This demo uses it as metadata storage for Trino and Spark. ** *Open policy agent* (OPA): An open-source, general-purpose policy engine unifies policy enforcement across the stack. @@ -52,7 +52,7 @@ This demo will You can see the deployed products and their relationship in the following diagram: -image::spark-k8s-anomaly-detection-taxi-data/overview.png[] +image::spark-k8s-anomaly-detection-taxi-data/overview.drawio.png[] == List the deployed Stackable services @@ -74,34 +74,25 @@ $ stackablectl stacklet list │ superset ┆ superset ┆ default ┆ node-http http://superset-node.default.svc.cluster.local:8088 ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ trino ┆ trino ┆ default ┆ coordinator-https https://trino-coordinator.default.svc.cluster.local:8443 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ minio ┆ minio-console ┆ default ┆ https https://10.0.0.11:31142 ┆ │ └──────────┴───────────────┴───────────┴─────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- include::partial$instance-hint.adoc[] -== MinIO +== S3 === List Buckets -The S3 provided by MinIO is used as persistent storage to store all the data used. -Open the endpoint `https` retrieved by `stackablectl stacklet list` in your browser (https://10.0.0.11:31142 in this case). -You need to accept the self-signed certificate in your browser before you can access the MinIO console. -If the console is not reachable, you can forward the port used by the MinIO console and use https://localhost:9001 instead. +The S3 provided by Garage is used as persistent storage to store all the data used. + +include::partial$inspect-s3.adoc[] [source,console] ---- -$ kubectl port-forward service/minio-console 9001:https +$ rclone lsd :s3: ---- -image::spark-k8s-anomaly-detection-taxi-data/minio_0.png[] - -Log in with the username `admin` and password `adminadmin`. - -image::spark-k8s-anomaly-detection-taxi-data/minio_2.png[] - -Here, you can see the two buckets the S3 is split into: +There are two buckets: . `demo`: The demo loads static datasets into this area. It is stored in parquet format. It forms the basis for the model that Spark will train. @@ -110,17 +101,21 @@ Here, you can see the two buckets the S3 is split into: === Inspect raw data -Click on the bucket `demo` and then on `ny-taxi-data` and `raw` respectively. - -image::spark-k8s-anomaly-detection-taxi-data/minio_3.png[] +[source,console] +---- +$ rclone lsl :s3:demo/ny-taxi-data/raw +---- -This folder (called prefixes in S3) contains a dataset of similarly structured data files. +This prefix contains a dataset of similarly structured data files. The data is partitioned by month and contains several hundred MBs, which may seem small for a dataset. Still, the model is a time-series model where the data has decreasing relevance the "older" it is, especially when the data is subject to multiple external factors, many of which are unknown and fluctuating in scope and effect. -The second bucket prediction contains the output from the model scoring process under `prediction/anomaly-detection/iforest/data`: +The second bucket `prediction` contains the output from the model scoring process: -image::spark-k8s-anomaly-detection-taxi-data/minio_4.png[] +[source,console] +---- +$ rclone lsl :s3:prediction/anomaly-detection +---- This is a much smaller file, as it only contains scores for each aggregated period. diff --git a/docs/modules/demos/pages/trino-iceberg.adoc b/docs/modules/demos/pages/trino-iceberg.adoc index 8f9226a2..13588f10 100644 --- a/docs/modules/demos/pages/trino-iceberg.adoc +++ b/docs/modules/demos/pages/trino-iceberg.adoc @@ -56,18 +56,24 @@ $ stackablectl stacklet list │ opa ┆ opa ┆ default ┆ ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ trino ┆ trino ┆ default ┆ coordinator-https https://172.18.0.2:30856 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ minio ┆ minio-console ┆ default ┆ http http://172.18.0.2:32489 ┆ │ └─────────┴───────────────┴───────────┴───────────────────────────────────────────────┴─────────────────────────────────┘ ---- include::partial$instance-hint.adoc[] -== MinIO +== S3 + +include::partial$inspect-s3.adoc[] + +There is a single bucket, `lakehouse`: + +[source,console] +---- +$ rclone lsd :s3: +---- -You can view the available buckets and objects (think of files) described in the xref:data-lakehouse-iceberg-trino-spark.adoc#_minio[data-lakehouse-iceberg-trino-spark] demo. -Currently, the bucket `lakehouse` is still empty, but will be filled during the next steps. +It is still empty at this point, but will be filled during the next steps. == Connect to Trino diff --git a/docs/modules/demos/pages/trino-taxi-data.adoc b/docs/modules/demos/pages/trino-taxi-data.adoc index a4ec1de2..167836d5 100644 --- a/docs/modules/demos/pages/trino-taxi-data.adoc +++ b/docs/modules/demos/pages/trino-taxi-data.adoc @@ -1,5 +1,5 @@ = trino-taxi-data -:description: Install and demo Trino with NYC taxi data: Query with SQL, visualize with Superset, and explore data in MinIO and Trino on Kubernetes. +:description: Install and demo Trino with NYC taxi data: Query with SQL, visualize with Superset, and explore data in S3 and Trino on Kubernetes. :superset-docs: https://superset.apache.org/docs/creating-charts-dashboards/creating-your-first-dashboard#creating-charts-in-explore-view :nyc-website: https://www1.nyc.gov/assets/tlc/downloads/pdf/data_dictionary_trip_records_yellow.pdf @@ -40,7 +40,7 @@ This demo will Trino via SQL queries and build dashboards on top of that data. ** *Trino*: A fast distributed SQL query engine for big data analytics that helps you explore your data universe. This demo uses it to enable SQL access to the data. -** *MinIO*: A S3 compatible object store. This demo uses it as persistent storage to store all the data used +** *Garage*: S3 compatible object store. This demo uses it as persistent storage to store all the data used ** *Hive metastore*: A service that stores metadata related to Apache Hive and other services. This demo uses it as metadata storage for Trino. ** *Open policy agent* (OPA): An open-source, general-purpose policy engine unifying policy enforcement across the @@ -51,7 +51,7 @@ This demo will You can see the deployed products and their relationship in the following diagram: -image::trino-taxi-data/overview.png[] +image::trino-taxi-data/overview.drawio.png[] == List the deployed Stackable services @@ -71,8 +71,6 @@ $ stackablectl stacklet list │ superset ┆ superset ┆ default ┆ node-http http://172.18.0.2:31312 ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ trino ┆ trino ┆ default ┆ coordinator-https https://172.18.0.2:30755 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ minio ┆ minio-console ┆ default ┆ https https://172.18.0.2:32654 ┆ │ └──────────┴───────────────┴───────────┴──────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- @@ -80,20 +78,16 @@ include::partial$instance-hint.adoc[] == Inspect the data in S3 -The S3 provided by MinIO is used as a persistent storage to store all the data used. You can look at the test data -within the MinIO web interface by opening the endpoint `http` from your `stackablectl stacklet list` command -output. You have to use the endpoint from your command output. In this case, it is https://172.18.0.2:32654. Open it with -your favourite browser. - -image::trino-taxi-data/minio_1.png[] - -Log in with the username `admin` and password `adminadmin`. +The S3 provided by Garage is used as a persistent storage to store all the data used. -image::trino-taxi-data/minio_2.png[] +include::partial$inspect-s3.adoc[] -Click on the bucket `demo` and open the folders `ny-taxi-data` -> `raw`. +The test data is in the `demo` bucket, under `ny-taxi-data/raw`: -image::trino-taxi-data/minio_3.png[] +[source,console] +---- +$ rclone lsl :s3:demo/ny-taxi-data/raw +---- The demo uploaded 1GB of parquet files, one file per month. The data contains taxi rides in New York City. The file size (and therefore the number of rides) decreased drastically because of the COVID-19 pandemic starting from `2020-03`. @@ -305,7 +299,7 @@ You also have the possibility to create additional charts and bundle them togeth === Load Additional Data -You can use the MinIO webinterface to upload additional data. As an alternative you can use the S3 API with an S3 client like https://s3tools.org/s3cmd[s3cmd]. +You can upload additional data with any S3 client, such as the AWS CLI or `rclone` configured as described above. It is recommended to put the data into a folder (prefix) in the `demo` bucket. Have a look at the defined tables inside the `hive.demo` schema on how to inform Trino about the newly available data. diff --git a/docs/modules/demos/partials/inspect-s3.adoc b/docs/modules/demos/partials/inspect-s3.adoc new file mode 100644 index 00000000..48f78235 --- /dev/null +++ b/docs/modules/demos/partials/inspect-s3.adoc @@ -0,0 +1,61 @@ +Garage has no web interface, so the buckets are inspected with an S3 client. + +Read the endpoint and credentials out of the cluster: + +[source,console] +---- +$ KUBECTL_NAMESPACE="default" # or whichever you set when running stackablectl demo install +$ export S3_ENDPOINT="https://$(kubectl -n "$KUBECTL_NAMESPACE" get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'):$(kubectl -n "$KUBECTL_NAMESPACE" get service garage -o jsonpath='{.spec.ports[0].nodePort}')" +$ export S3_ACCESS_KEY="$(kubectl -n "$KUBECTL_NAMESPACE" get secret s3-credentials -o jsonpath='{.data.accessKey}' | base64 -d)" +$ export S3_SECRET_KEY="$(kubectl -n "$KUBECTL_NAMESPACE" get secret s3-credentials -o jsonpath='{.data.secretKey}' | base64 -d)" +$ export S3_REGION=region-1 +---- + +The region is not optional. +Garage rejects any request signed for a different region than the one it is configured with, and most clients fall back to an AWS region when none is set. + +The certificate is issued by the Stackable secret-operator, whose CA your machine does not trust, so the commands below skip certificate verification. + +[tabs] +==== +AWS CLI:: ++ +-- +[source,console] +---- +$ export AWS_ENDPOINT_URL="$S3_ENDPOINT" +$ export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" +$ export AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" +$ export AWS_DEFAULT_REGION="$S3_REGION" +$ aws configure set default.s3.addressing_style path +---- + +The commands below are written for `rclone`. +The `aws` equivalents are `aws s3 ls` to list buckets and `aws s3 ls s3:/// --recursive --human-readable` to list objects, both with `--no-verify-ssl`. +-- + +rclone:: ++ +-- +`rclone` needs no installation, it runs straight from a container image: + +[source,console] +---- +$ rclone() { + docker run --rm -i --network host \ + -e RCLONE_S3_PROVIDER=Other \ + -e RCLONE_S3_ENDPOINT="$S3_ENDPOINT" \ + -e RCLONE_S3_REGION="$S3_REGION" \ + -e RCLONE_S3_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ + -e RCLONE_S3_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ + -e RCLONE_S3_FORCE_PATH_STYLE=true \ + -e RCLONE_NO_CHECK_CERTIFICATE=true \ + rclone/rclone:1.75.1 "$@" + } +---- + +`--network host` is there for anyone trying the demos out on a local cluster, where the endpoint is only reachable from the host itself. + +In the commands below, `lsd` lists buckets, `lsf` lists names only, and `lsl` adds sizes and modification times. +-- +==== diff --git a/stacks/_templates/garage.yaml b/stacks/_templates/garage.yaml new file mode 100644 index 00000000..fc1e79c1 --- /dev/null +++ b/stacks/_templates/garage.yaml @@ -0,0 +1,344 @@ +--- +# Garage provides the S3 implementation for the demo stacks. +# +# Garage terminates no TLS on any of its endpoints, so an nginx sidecar does it +# and forwards to Garage on loopback. +# +# Parameters (see the stack definition in stacks-v2.yaml): +# garageBuckets space separated list of buckets to create +# garageStorageSize size of the PVC backing the object data +apiVersion: v1 +kind: ConfigMap +metadata: + name: garage + labels: + stackable.tech/vendor: Stackable +data: + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + garage.toml: | + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "lmdb" + + # Single node, no redundancy. This is a demo, not a production deployment. + replication_factor = 1 + + # Throwaway value, this cluster is never joined by another node. + rpc_secret = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + # Garage rejects requests signed for a different region, so this must match + # the region every client signs with: the S3Connection (and therefore the + # products configured by the operators), the load jobs, and any NiFi flow. + # The name is a provider agnostic one, deliberately not an AWS region: the + # Hadoop S3 implementation falls back to us-east-2 when none is set, so + # leaving it unset would silently sign with the wrong region. + s3_region = "region-1" + api_bind_addr = "[::]:3900" + + [admin] + api_bind_addr = "[::]:3903" + # Garage terminates no TLS, so nginx does it and forwards to Garage on + # loopback. Mounted over /etc/nginx/nginx.conf, hence the full file. + nginx.conf: | + events {} + http { + server { + listen 9000 ssl; + ssl_certificate /stackable/tls/tls.crt; + ssl_certificate_key /stackable/tls/tls.key; + + # Don't buffer or size-limit object uploads. + client_max_body_size 0; + + location / { + proxy_pass http://127.0.0.1:3900; + # Must be the original Host, it is part of the SigV4 signature. + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_buffering off; + proxy_request_buffering off; + } + } + } +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: garage + labels: + stackable.tech/vendor: Stackable +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ garageStorageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: garage + labels: + stackable.tech/vendor: Stackable +spec: + # NodePort so the S3 API can be reached from outside the cluster, e.g. to + # inspect the buckets with `aws s3 ls` or `rclone lsf`. + type: NodePort + selector: + app: garage + ports: + - name: https + port: 9000 + targetPort: https +--- +# Garage's own S3 port, without the nginx TLS layer in front of it. +# +# This exists so that notebooks can talk to S3 with a plain endpoint and no +# trust store boilerplate. Verifying the secret-operator CA from a notebook +# means building a JVM trust store for Spark and shipping it to the executors, +# which drowns the actual content in TLS plumbing. Products configured by the +# operators use the TLS endpoint on the "garage" Service instead. +# +# Deliberately a separate ClusterIP Service: every port of a NodePort Service is +# reachable from outside the cluster, and this one must not be. +apiVersion: v1 +kind: Service +metadata: + name: garage-plaintext + labels: + stackable.tech/vendor: Stackable +spec: + type: ClusterIP + selector: + app: garage + ports: + - name: http + port: 3900 + targetPort: s3 +--- +# Prometheus metrics, served by Garage's admin API on /metrics. +apiVersion: v1 +kind: Service +metadata: + name: garage-metrics + labels: + stackable.tech/vendor: Stackable + app: garage +spec: + type: ClusterIP + selector: + app: garage + ports: + - name: metrics + port: 3903 + targetPort: admin +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garage + labels: + app: garage + stackable.tech/vendor: Stackable +spec: + replicas: 1 + strategy: + # The PVC is ReadWriteOnce, so the old Pod has to release it before the new + # one can claim it. + type: Recreate + selector: + matchLabels: + app: garage + template: + metadata: + labels: + app: garage + stackable.tech/vendor: Stackable + spec: + containers: + - name: garage + image: oci.stackable.tech/stackable/dxflrs/garage:v2.4.1 + imagePullPolicy: IfNotPresent + # The image has no entrypoint. --single-node creates the cluster + # layout, --default-access-key creates the access key from the + # GARAGE_DEFAULT_* variables below. That key is allowed to create + # buckets, which is how the garage-init-buckets Job below creates + # every bucket the stack needs. + command: + - /garage + - server + - --single-node + - --default-access-key + env: + # Garage requires the GK<24 hex> key id format and a 64 hex char + # secret, so this is not a free form password. + # Change it here, in the garage-init-buckets Job below, and in the + # s3-connection.yaml of each stack. + - name: GARAGE_DEFAULT_ACCESS_KEY + value: GK31c0ffee31c0ffee31c0ffee + - name: GARAGE_DEFAULT_SECRET_KEY + value: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef + ports: + - name: s3 + containerPort: 3900 + - name: rpc + containerPort: 3901 + - name: admin + containerPort: 3903 + readinessProbe: + httpGet: + path: /health + port: admin + volumeMounts: + - name: config + mountPath: /etc/garage.toml + subPath: garage.toml + - name: data + mountPath: /var/lib/garage + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + - name: nginx + image: docker.io/library/nginx:1.29-alpine + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 9000 + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: tls + mountPath: /stackable/tls + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: config + configMap: + name: garage + - name: data + persistentVolumeClaim: + claimName: garage + # Request a TLS certificate from the secret-operator for nginx to serve + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=garage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +--- +# Creates every bucket the stack needs. The access key created by +# --default-bucket is allowed to create buckets, so this needs no admin +# credentials, only the ordinary S3 key. `rclone mkdir` is idempotent, so +# re-running the Job is harmless. +apiVersion: batch/v1 +kind: Job +metadata: + name: garage-init-buckets + labels: + stackable.tech/vendor: Stackable +spec: + template: + metadata: + labels: + stackable.tech/vendor: Stackable + spec: + containers: + - name: init-buckets + image: docker.io/rclone/rclone:1.75.1 + # The image entrypoint is rclone itself, so override it for the loop. + command: + - /bin/sh + - -euo + - pipefail + - -c + - | + # Wait for Garage to accept requests. The Deployment readiness + # probe covers Garage itself, but not the nginx sidecar in front. + # Bounded, so a broken endpoint fails the Job instead of hanging. + attempts=0 + until rclone lsd :s3: >/dev/null 2>&1; do + attempts=$((attempts + 1)) + if [ "$attempts" -gt 150 ]; then + echo "Garage did not become available within 5 minutes" >&2 + rclone lsd :s3: # run once more to surface the actual error + exit 1 + fi + echo "Waiting for Garage to become available..." + sleep 2 + done + + for bucket in {{ garageBuckets }}; do + echo "Creating bucket '$bucket'" + rclone mkdir ":s3:$bucket" + done + + rclone lsd :s3: + env: + # No config file, everything is configured through the environment. + - name: RCLONE_CONFIG + value: /dev/null + - name: RCLONE_S3_PROVIDER + value: Other + - name: RCLONE_S3_ENDPOINT + value: https://garage.{{ NAMESPACE }}.svc.cluster.local:9000 + # Must match s3_region in the Garage config above. + - name: RCLONE_S3_REGION + value: region-1 + # Garage only serves path style, not virtual-hosted style. + - name: RCLONE_S3_FORCE_PATH_STYLE + value: "true" + - name: RCLONE_CA_CERT + value: /stackable/tls/ca.crt + # Must match the GARAGE_DEFAULT_* variables in the Deployment above. + - name: RCLONE_S3_ACCESS_KEY_ID + value: GK31c0ffee31c0ffee31c0ffee + - name: RCLONE_S3_SECRET_ACCESS_KEY + value: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef + volumeMounts: + - name: tls + mountPath: /stackable/tls + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/provision-parts: public + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + restartPolicy: OnFailure + backoffLimit: 50 diff --git a/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml b/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml deleted file mode 100644 index 35d2fe7c..00000000 --- a/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml +++ /dev/null @@ -1,724 +0,0 @@ ---- -# Source: minio/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "minio-sa" ---- -# Source: minio/templates/secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- -# Source: minio/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -data: - initialize: |- - #!/bin/sh - set -e # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 - LIMIT=29 # Allow 30 attempts - set -e # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) - SECRET=$(cat /config/rootPassword) - set +e # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" - $MC_COMMAND - STATUS=$? - until [ $STATUS = 0 ]; do - ATTEMPTS=$(expr $ATTEMPTS + 1) - echo \"Failed attempts: $ATTEMPTS\" - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 - fi - sleep 2 # 1 second intervals between attempts - $MC_COMMAND - STATUS=$? - done - set -e # reset `e` as active - return 0 - } - - # checkBucketExists ($bucket) - # Check if the bucket exists, by using the exit code of `mc ls` - checkBucketExists() { - BUCKET=$1 - CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) - return $? - } - - # createBucket ($bucket, $policy, $purge) - # Ensure bucket exists, purging if asked to - createBucket() { - BUCKET=$1 - POLICY=$2 - PURGE=$3 - VERSIONING=$4 - OBJECTLOCKING=$5 - - # Purge the bucket, if set & exists - # Since PURGE is user input, check explicitly for `true` - if [ $PURGE = true ]; then - if checkBucketExists $BUCKET; then - echo "Purging bucket '$BUCKET'." - set +e # don't exit if this fails - ${MC} rm -r --force myminio/$BUCKET - set -e # reset `e` as active - else - echo "Bucket '$BUCKET' does not exist, skipping purge." - fi - fi - - # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) - if ! checkBucketExists $BUCKET; then - if [ ! -z $OBJECTLOCKING ]; then - if [ $OBJECTLOCKING = true ]; then - echo "Creating bucket with OBJECTLOCKING '$BUCKET'" - ${MC} mb --with-lock myminio/$BUCKET - elif [ $OBJECTLOCKING = false ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - fi - elif [ -z $OBJECTLOCKING ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - else - echo "Bucket '$BUCKET' already exists." - fi - fi - - # set versioning for bucket if objectlocking is disabled or not set - if [ $OBJECTLOCKING = false ]; then - if [ ! -z $VERSIONING ]; then - if [ $VERSIONING = true ]; then - echo "Enabling versioning for '$BUCKET'" - ${MC} version enable myminio/$BUCKET - elif [ $VERSIONING = false ]; then - echo "Suspending versioning for '$BUCKET'" - ${MC} version suspend myminio/$BUCKET - fi - fi - else - echo "Bucket '$BUCKET' versioning unchanged." - fi - - # At this point, the bucket should exist, skip checking for existence - # Set policy on the bucket - echo "Setting policy of bucket '$BUCKET' to '$POLICY'." - ${MC} anonymous set $POLICY myminio/$BUCKET - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - # Create the buckets - createBucket lakehouse "public" false false false - - add-user: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkUserExists () - # Check if the user exists, by using the exit code of `mc admin user info` - checkUserExists() { - CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createUser ($policy) - createUser() { - POLICY=$1 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the user if it does not exist - if ! checkUserExists ; then - echo "Creating user '$USER'" - cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio - else - echo "User '$USER' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - - # set policy for user - if [ ! -z $POLICY -a $POLICY != " " ] ; then - echo "Adding policy '$POLICY' for '$USER'" - set +e ; # policy already attach errors out, allow it. - ${MC} admin policy attach myminio $POLICY --user=$USER - set -e - else - echo "User '$USER' has no policy attached." - fi - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - # Create the users - echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP - echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP - createUser consoleAdmin - - add-policy: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkPolicyExists ($policy) - # Check if the policy exists, by using the exit code of `mc admin policy info` - checkPolicyExists() { - POLICY=$1 - CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) - return $? - } - - # createPolicy($name, $filename) - createPolicy () { - NAME=$1 - FILENAME=$2 - - # Create the name if it does not exist - echo "Checking policy: $NAME (in /config/$FILENAME.json)" - if ! checkPolicyExists $NAME ; then - echo "Creating policy '$NAME'" - else - echo "Policy '$NAME' already exists." - fi - ${MC} admin policy create myminio $NAME /config/$FILENAME.json - - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - add-svcacct: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 2 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkSvcacctExists () - # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` - checkSvcacctExists() { - CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createSvcacct ($user) - createSvcacct () { - USER=$1 - FILENAME=$2 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the svcacct if it does not exist - if ! checkSvcacctExists ; then - echo "Creating svcacct '$SVCACCT'" - # Check if policy file is define - if [ -z $FILENAME ]; then - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER - else - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER - fi - else - echo "Svcacct '$SVCACCT' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - custom-command: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # runCommand ($@) - # Run custom mc command - runCommand() { - ${MC} "$@" - return $? - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme ---- -# Source: minio/templates/console-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio-console - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: minio - release: minio ---- -# Source: minio/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - monitoring: "true" -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -# Source: minio/templates/statefulset.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio-svc - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - publishNotReadyAddresses: true - clusterIP: None - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -# Source: minio/templates/statefulset.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable -spec: - updateStrategy: - type: RollingUpdate - podManagementPolicy: "Parallel" - serviceName: minio-svc - replicas: 2 - selector: - matchLabels: - app: minio - release: minio - template: - metadata: - name: minio - labels: - app: minio - release: minio - stackable.tech/vendor: Stackable - annotations: - checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 - checksum/config: 71cbcb27e07d5e839d7783fb29f329459b370e7012c87676ede572c147553dbb - spec: - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - runAsGroup: 1000 - runAsUser: 1000 - serviceAccountName: minio-sa - containers: - - name: minio - image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z - imagePullPolicy: IfNotPresent - command: - - /bin/sh - - -ce - - | - # minio requires the TLS key pair to be specially named - # mkdir -p /etc/minio/certs - cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt - cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key - - # Copy the CA cert from the "tls" SecretClass - mkdir -p /etc/minio/certs/CAs - cp -v /etc/minio/original_certs/ca.crt /etc/minio/certs/CAs/public.crt - - /usr/bin/docker-entrypoint.sh minio server https://minio-{0...1}.minio-svc.{{ NAMESPACE }}.svc.cluster.local/export -S /etc/minio/certs/ --address :9000 --console-address :9001 - volumeMounts: - - name: export - mountPath: /export - - mountPath: /etc/minio/original_certs - name: tls - - mountPath: /etc/minio/certs - name: certs - ports: - - name: https - containerPort: 9000 - - name: https-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: rootUser - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: rootPassword - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - resources: - requests: - cpu: 500m - memory: 1Gi - securityContext: - readOnlyRootFilesystem: false - volumes: - - name: minio-user - secret: - secretName: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio,pod - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - volumeClaimTemplates: - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: export - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 50Gi ---- -# Source: minio/templates/post-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job - labels: - app: minio-post-job - chart: minio-5.4.0 - release: minio - heritage: Helm - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation -spec: - template: - metadata: - labels: - app: minio-job - release: minio - stackable.tech/vendor: Stackable - spec: - restartPolicy: OnFailure - volumes: - - name: etc-path - emptyDir: {} - - name: tmp - emptyDir: {} - - name: minio-configuration - projected: - sources: - - configMap: - name: minio - - secret: - name: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio,pod - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - serviceAccountName: minio-sa - containers: - - name: minio-make-bucket - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - . /config/initialize - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi - - name: minio-make-user - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - . /config/add-user - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi diff --git a/stacks/_templates/minio-distributed-small-tls/values.yaml b/stacks/_templates/minio-distributed-small-tls/values.yaml deleted file mode 100644 index 9fbdb1e6..00000000 --- a/stacks/_templates/minio-distributed-small-tls/values.yaml +++ /dev/null @@ -1,99 +0,0 @@ ---- -additionalLabels: - stackable.tech/vendor: Stackable - stackable.tech/stack: "{{ STACK }}" - # The following is a Jinja2 template directive, not a regular comment. - # The `#` prefix is required to prevent YAML parsing errors, but Jinja2 still processes it. - # {% if DEMO is defined %} - stackable.tech/demo: "{{ DEMO }}" - # {% endif %} -podLabels: - stackable.tech/vendor: Stackable - stackable.tech/stack: "{{ STACK }}" - # The following is a Jinja2 template directive, not a regular comment. - # The `#` prefix is required to prevent YAML parsing errors, but Jinja2 still processes it. - # {% if DEMO is defined %} - stackable.tech/demo: "{{ DEMO }}" - # {% endif %} -rootUser: admin -rootPassword: adminadmin -mode: distributed -replicas: 2 -persistence: - size: 50Gi -buckets: - - name: lakehouse - policy: public -resources: - requests: - cpu: 500m - memory: 1Gi -service: - type: NodePort - nodePort: null -consoleService: - type: NodePort - nodePort: null -tls: - enabled: true -extraVolumes: - # Request a TLS certificate from the secret-operator - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio,pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory -extraVolumeMounts: - # Mount the certificate generated by the secret-operator - - name: tls - mountPath: /etc/minio/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /etc/minio/certs - name: certs - -customCommandJob: - extraVolumes: - # Request a TLS certificate from the secret-operator - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio,pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory - # WARNING: this is currently only used by the custom-scripts job container. Other containers do not mount these. - extraVolumeMounts: - # Mount the certificate generated by the secret-operator - - name: tls - mountPath: /etc/minio/mc/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /etc/minio/mc/certs/CAs - name: certs diff --git a/stacks/_templates/minio-distributed-tls/README.md b/stacks/_templates/minio-distributed-tls/README.md deleted file mode 100644 index 7a3f0ac8..00000000 --- a/stacks/_templates/minio-distributed-tls/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# MinIO with TLS from secret-operator - -MinIO has a severe limitation whereby the TLS certificates must be named `public.crt` -and `private.key`. This goes against Kubernetes naming of `tls.crt` and `tls.key`. - -The upstream minio chart is also too limited: - -- No way to add initContainers (to rename cert files in a shared volume). -- No way to edit the container command (to rename cert files before starting minio). - -Therefore, we will render the upstream chart here, and then apply the necessary -customizations on top. - -```bash -helm repo add minio https://charts.min.io/ -helm template minio minio/minio -f values.yaml > rendered-chart.yaml -``` diff --git a/stacks/_templates/minio-distributed-tls/rendered-chart.yaml b/stacks/_templates/minio-distributed-tls/rendered-chart.yaml deleted file mode 100644 index 249048b5..00000000 --- a/stacks/_templates/minio-distributed-tls/rendered-chart.yaml +++ /dev/null @@ -1,714 +0,0 @@ ---- -# Source: minio/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "minio-sa" ---- -# Source: minio/templates/secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- -# Source: minio/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -data: - initialize: |- - #!/bin/sh - set -e # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 - LIMIT=29 # Allow 30 attempts - set -e # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) - SECRET=$(cat /config/rootPassword) - set +e # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" - $MC_COMMAND - STATUS=$? - until [ $STATUS = 0 ]; do - ATTEMPTS=$(expr $ATTEMPTS + 1) - echo \"Failed attempts: $ATTEMPTS\" - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 - fi - sleep 2 # 1 second intervals between attempts - $MC_COMMAND - STATUS=$? - done - set -e # reset `e` as active - return 0 - } - - # checkBucketExists ($bucket) - # Check if the bucket exists, by using the exit code of `mc ls` - checkBucketExists() { - BUCKET=$1 - CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) - return $? - } - - # createBucket ($bucket, $policy, $purge) - # Ensure bucket exists, purging if asked to - createBucket() { - BUCKET=$1 - POLICY=$2 - PURGE=$3 - VERSIONING=$4 - OBJECTLOCKING=$5 - - # Purge the bucket, if set & exists - # Since PURGE is user input, check explicitly for `true` - if [ $PURGE = true ]; then - if checkBucketExists $BUCKET; then - echo "Purging bucket '$BUCKET'." - set +e # don't exit if this fails - ${MC} rm -r --force myminio/$BUCKET - set -e # reset `e` as active - else - echo "Bucket '$BUCKET' does not exist, skipping purge." - fi - fi - - # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) - if ! checkBucketExists $BUCKET; then - if [ ! -z $OBJECTLOCKING ]; then - if [ $OBJECTLOCKING = true ]; then - echo "Creating bucket with OBJECTLOCKING '$BUCKET'" - ${MC} mb --with-lock myminio/$BUCKET - elif [ $OBJECTLOCKING = false ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - fi - elif [ -z $OBJECTLOCKING ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - else - echo "Bucket '$BUCKET' already exists." - fi - fi - - # set versioning for bucket if objectlocking is disabled or not set - if [ $OBJECTLOCKING = false ]; then - if [ ! -z $VERSIONING ]; then - if [ $VERSIONING = true ]; then - echo "Enabling versioning for '$BUCKET'" - ${MC} version enable myminio/$BUCKET - elif [ $VERSIONING = false ]; then - echo "Suspending versioning for '$BUCKET'" - ${MC} version suspend myminio/$BUCKET - fi - fi - else - echo "Bucket '$BUCKET' versioning unchanged." - fi - - # At this point, the bucket should exist, skip checking for existence - # Set policy on the bucket - echo "Setting policy of bucket '$BUCKET' to '$POLICY'." - ${MC} anonymous set $POLICY myminio/$BUCKET - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - # Create the buckets - createBucket staging "public" false false false - createBucket lakehouse "public" false false false - - add-user: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkUserExists () - # Check if the user exists, by using the exit code of `mc admin user info` - checkUserExists() { - CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createUser ($policy) - createUser() { - POLICY=$1 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the user if it does not exist - if ! checkUserExists ; then - echo "Creating user '$USER'" - cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio - else - echo "User '$USER' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - - # set policy for user - if [ ! -z $POLICY -a $POLICY != " " ] ; then - echo "Adding policy '$POLICY' for '$USER'" - set +e ; # policy already attach errors out, allow it. - ${MC} admin policy attach myminio $POLICY --user=$USER - set -e - else - echo "User '$USER' has no policy attached." - fi - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - # Create the users - echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP - echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP - createUser consoleAdmin - - add-policy: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkPolicyExists ($policy) - # Check if the policy exists, by using the exit code of `mc admin policy info` - checkPolicyExists() { - POLICY=$1 - CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) - return $? - } - - # createPolicy($name, $filename) - createPolicy () { - NAME=$1 - FILENAME=$2 - - # Create the name if it does not exist - echo "Checking policy: $NAME (in /config/$FILENAME.json)" - if ! checkPolicyExists $NAME ; then - echo "Creating policy '$NAME'" - else - echo "Policy '$NAME' already exists." - fi - ${MC} admin policy create myminio $NAME /config/$FILENAME.json - - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - add-svcacct: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 2 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkSvcacctExists () - # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` - checkSvcacctExists() { - CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createSvcacct ($user) - createSvcacct () { - USER=$1 - FILENAME=$2 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the svcacct if it does not exist - if ! checkSvcacctExists ; then - echo "Creating svcacct '$SVCACCT'" - # Check if policy file is define - if [ -z $FILENAME ]; then - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER - else - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER - fi - else - echo "Svcacct '$SVCACCT' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - custom-command: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # runCommand ($@) - # Run custom mc command - runCommand() { - ${MC} "$@" - return $? - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme ---- -# Source: minio/templates/console-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio-console - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: minio - release: minio ---- -# Source: minio/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - monitoring: "true" -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -# Source: minio/templates/statefulset.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio-svc - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - publishNotReadyAddresses: true - clusterIP: None - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -# Source: minio/templates/statefulset.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable -spec: - updateStrategy: - type: RollingUpdate - podManagementPolicy: "Parallel" - serviceName: minio-svc - replicas: 5 - selector: - matchLabels: - app: minio - release: minio - template: - metadata: - name: minio - labels: - app: minio - release: minio - stackable.tech/vendor: Stackable - annotations: - checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 - checksum/config: 66f252598ba8542f924dd76a5da7f64ae6a943b95798d752c65d32214320cbb4 - spec: - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - runAsGroup: 1000 - runAsUser: 1000 - serviceAccountName: minio-sa - containers: - - name: minio - image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # minio requires the TLS key pair to be specially named - mkdir -p /etc/minio/certs/CAs - cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt - cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key - cp -v /etc/minio/original_certs/ca.crt /etc/minio/certs/CAs/ca.crt - - /usr/bin/docker-entrypoint.sh minio server https://minio-{0...4}.minio-svc.{{ NAMESPACE }}.svc.cluster.local/export -S /etc/minio/certs/ --address :9000 --console-address :9001 - volumeMounts: - - name: export - mountPath: /export - - mountPath: /etc/minio/original_certs - name: minio-tls - - mountPath: /etc/minio/certs - name: certs - ports: - - name: https - containerPort: 9000 - - name: https-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: rootUser - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: rootPassword - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - resources: - requests: - cpu: 500m - memory: 2Gi - securityContext: - readOnlyRootFilesystem: false - volumes: - - name: minio-user - secret: - secretName: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio,pod - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" - storageClassName: secrets.stackable.tech - name: minio-tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - volumeClaimTemplates: - - apiVersion: v1 - kind: PersistentVolumeClaim - metadata: - name: export - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 250Gi ---- -# Source: minio/templates/post-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job - labels: - app: minio-post-job - chart: minio-5.4.0 - release: minio - heritage: Helm - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation -spec: - template: - metadata: - labels: - app: minio-job - release: minio - stackable.tech/vendor: Stackable - spec: - restartPolicy: OnFailure - volumes: - - name: etc-path - emptyDir: {} - - name: tmp - emptyDir: {} - - name: minio-configuration - projected: - sources: - - configMap: - name: minio - - secret: - name: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio,pod - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "1" - storageClassName: secrets.stackable.tech - name: minio-tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - serviceAccountName: minio-sa - containers: - - name: minio-make-bucket - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - . /config/initialize - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: minio-tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi - - name: minio-make-user - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: [ "/bin/sh", "/config/add-user" ] - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: minio-tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi diff --git a/stacks/_templates/minio-distributed-tls/values.yaml b/stacks/_templates/minio-distributed-tls/values.yaml deleted file mode 100644 index d601af57..00000000 --- a/stacks/_templates/minio-distributed-tls/values.yaml +++ /dev/null @@ -1,100 +0,0 @@ ---- -additionalLabels: - stackable.tech/vendor: Stackable - stackable.tech/stack: "{{ STACK }}" - # The following is a Jinja2 template directive, not a regular comment. - # The `#` prefix is required to prevent YAML parsing errors, but Jinja2 still processes it. - # {% if DEMO is defined %} - stackable.tech/demo: "{{ DEMO }}" - # {% endif %} -podLabels: - stackable.tech/vendor: Stackable - stackable.tech/stack: "{{ STACK }}" - # The following is a Jinja2 template directive, not a regular comment. - # The `#` prefix is required to prevent YAML parsing errors, but Jinja2 still processes it. - # {% if DEMO is defined %} - stackable.tech/demo: "{{ DEMO }}" - # {% endif %} -rootUser: admin -rootPassword: adminadmin -mode: distributed -replicas: 5 -persistence: - size: 250Gi -buckets: - - name: staging - policy: public - - name: lakehouse - policy: public -resources: - requests: - cpu: 500m - memory: 2Gi -service: - type: NodePort - nodePort: null -consoleService: - type: NodePort - nodePort: null -tls: - enabled: true -extraVolumes: - # Request a TLS certificate from the secret-operator - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio,pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory -extraVolumeMounts: - # Mount the certificate generated by the secret-operator - - name: minio-tls - mountPath: /etc/minio/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /etc/minio/certs - name: certs -customCommandJob: - extraVolumes: - # Request a TLS certificate from the secret-operator - - name: minio-tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio,pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory - # WARNING: this is currently only used by the custom-scripts job container. Other containers do not mount these. - extraVolumeMounts: - # Mount the certificate generated by the secret-operator - - name: minio-tls - mountPath: /etc/minio/mc/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /etc/minio/mc/certs/CAs - name: certs diff --git a/stacks/_templates/minio-tls/README.md b/stacks/_templates/minio-tls/README.md deleted file mode 100644 index a007da87..00000000 --- a/stacks/_templates/minio-tls/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# MinIO with TLS from secret-operator - -MinIO has a severe limitation whereby the TLS certificates must be named `public.crt` -and `private.key`. This goes against Kubernetes naming of `tls.crt` and `tls.key`. - -The upstream minio chart is also too limited: - -- No way to add initContainers (to rename cert files in a shared volume). -- No way to edit the container command (to rename cert files before starting minio). - -Therefore, we will render the upstream chart here, and then apply the necessary -customizations on top. - -```yaml -helm template minio minio/minio -f values.yaml > rendered-chart.yaml -``` diff --git a/stacks/_templates/minio-tls/rendered-chart.yaml b/stacks/_templates/minio-tls/rendered-chart.yaml deleted file mode 100644 index 21741bc5..00000000 --- a/stacks/_templates/minio-tls/rendered-chart.yaml +++ /dev/null @@ -1,717 +0,0 @@ ---- -# Source: minio/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "minio-sa" ---- -# Source: minio/templates/secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- -# Source: minio/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -data: - initialize: |- - #!/bin/sh - set -e # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 - LIMIT=29 # Allow 30 attempts - set -e # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) - SECRET=$(cat /config/rootPassword) - set +e # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" - $MC_COMMAND - STATUS=$? - until [ $STATUS = 0 ]; do - ATTEMPTS=$(expr $ATTEMPTS + 1) - echo \"Failed attempts: $ATTEMPTS\" - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 - fi - sleep 2 # 1 second intervals between attempts - $MC_COMMAND - STATUS=$? - done - set -e # reset `e` as active - return 0 - } - - # checkBucketExists ($bucket) - # Check if the bucket exists, by using the exit code of `mc ls` - checkBucketExists() { - BUCKET=$1 - CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) - return $? - } - - # createBucket ($bucket, $policy, $purge) - # Ensure bucket exists, purging if asked to - createBucket() { - BUCKET=$1 - POLICY=$2 - PURGE=$3 - VERSIONING=$4 - OBJECTLOCKING=$5 - - # Purge the bucket, if set & exists - # Since PURGE is user input, check explicitly for `true` - if [ $PURGE = true ]; then - if checkBucketExists $BUCKET; then - echo "Purging bucket '$BUCKET'." - set +e # don't exit if this fails - ${MC} rm -r --force myminio/$BUCKET - set -e # reset `e` as active - else - echo "Bucket '$BUCKET' does not exist, skipping purge." - fi - fi - - # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) - if ! checkBucketExists $BUCKET; then - if [ ! -z $OBJECTLOCKING ]; then - if [ $OBJECTLOCKING = true ]; then - echo "Creating bucket with OBJECTLOCKING '$BUCKET'" - ${MC} mb --with-lock myminio/$BUCKET - elif [ $OBJECTLOCKING = false ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - fi - elif [ -z $OBJECTLOCKING ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - else - echo "Bucket '$BUCKET' already exists." - fi - fi - - # set versioning for bucket if objectlocking is disabled or not set - if [ $OBJECTLOCKING = false ]; then - if [ ! -z $VERSIONING ]; then - if [ $VERSIONING = true ]; then - echo "Enabling versioning for '$BUCKET'" - ${MC} version enable myminio/$BUCKET - elif [ $VERSIONING = false ]; then - echo "Suspending versioning for '$BUCKET'" - ${MC} version suspend myminio/$BUCKET - fi - fi - else - echo "Bucket '$BUCKET' versioning unchanged." - fi - - # At this point, the bucket should exist, skip checking for existence - # Set policy on the bucket - echo "Setting policy of bucket '$BUCKET' to '$POLICY'." - ${MC} anonymous set $POLICY myminio/$BUCKET - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - # Create the buckets - createBucket demo "public" false false false - - add-user: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkUserExists () - # Check if the user exists, by using the exit code of `mc admin user info` - checkUserExists() { - CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createUser ($policy) - createUser() { - POLICY=$1 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the user if it does not exist - if ! checkUserExists ; then - echo "Creating user '$USER'" - cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio - else - echo "User '$USER' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - - # set policy for user - if [ ! -z $POLICY -a $POLICY != " " ] ; then - echo "Adding policy '$POLICY' for '$USER'" - set +e ; # policy already attach errors out, allow it. - ${MC} admin policy attach myminio $POLICY --user=$USER - set -e - else - echo "User '$USER' has no policy attached." - fi - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - # Create the users - echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP - echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP - createUser consoleAdmin - - add-policy: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkPolicyExists ($policy) - # Check if the policy exists, by using the exit code of `mc admin policy info` - checkPolicyExists() { - POLICY=$1 - CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) - return $? - } - - # createPolicy($name, $filename) - createPolicy () { - NAME=$1 - FILENAME=$2 - - # Create the name if it does not exist - echo "Checking policy: $NAME (in /config/$FILENAME.json)" - if ! checkPolicyExists $NAME ; then - echo "Creating policy '$NAME'" - else - echo "Policy '$NAME' already exists." - fi - ${MC} admin policy create myminio $NAME /config/$FILENAME.json - - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - add-svcacct: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 2 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkSvcacctExists () - # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` - checkSvcacctExists() { - CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createSvcacct ($user) - createSvcacct () { - USER=$1 - FILENAME=$2 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the svcacct if it does not exist - if ! checkSvcacctExists ; then - echo "Creating svcacct '$SVCACCT'" - # Check if policy file is define - if [ -z $FILENAME ]; then - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER - else - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER - fi - else - echo "Svcacct '$SVCACCT' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - - custom-command: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # runCommand ($@) - # Run custom mc command - runCommand() { - ${MC} "$@" - return $? - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme ---- -# Source: minio/templates/pvc.yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: "10Gi" ---- -# Source: minio/templates/console-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio-console - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: minio - release: minio ---- -# Source: minio/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - monitoring: "true" -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -# Source: minio/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable -spec: - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 - replicas: 1 - selector: - matchLabels: - app: minio - release: minio - template: - metadata: - name: minio - labels: - app: minio - release: minio - stackable.tech/vendor: Stackable - annotations: - checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 - checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 - spec: - securityContext: - - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - runAsGroup: 1000 - runAsUser: 1000 - - serviceAccountName: minio-sa - containers: - - name: minio - image: "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # minio requires the TLS key pair to be specially named - # mkdir -p /etc/minio/certs - cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt - cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key - - /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 - volumeMounts: - - name: minio-user - mountPath: "/tmp/credentials" - readOnly: true - - name: export - mountPath: /export - - mountPath: /etc/minio/original_certs - name: tls - - mountPath: /etc/minio/certs - name: certs - ports: - - name: https - containerPort: 9000 - - name: https-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: rootUser - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: rootPassword - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - resources: - requests: - cpu: 1 - memory: 2Gi - securityContext: - readOnlyRootFilesystem: false - volumes: - - name: export - persistentVolumeClaim: - claimName: minio - - name: minio-user - secret: - secretName: minio - - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs ---- -# Source: minio/templates/post-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job - labels: - app: minio-post-job - chart: minio-5.4.0 - release: minio - heritage: Helm - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation -spec: - template: - metadata: - labels: - app: minio-job - release: minio - stackable.tech/vendor: Stackable - spec: - restartPolicy: OnFailure - volumes: - - name: etc-path - emptyDir: {} - - name: tmp - emptyDir: {} - - name: minio-configuration - projected: - sources: - - configMap: - name: minio - - secret: - name: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - serviceAccountName: minio-sa - containers: - - name: minio-make-bucket - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/initialize - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi - - name: minio-make-user - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/add-user - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi diff --git a/stacks/_templates/minio-tls/values.yaml b/stacks/_templates/minio-tls/values.yaml deleted file mode 100644 index e341d8ee..00000000 --- a/stacks/_templates/minio-tls/values.yaml +++ /dev/null @@ -1,86 +0,0 @@ ---- -additionalLabels: - stackable.tech/vendor: Stackable -podLabels: - stackable.tech/vendor: Stackable -rootUser: admin -rootPassword: adminadmin -mode: standalone -persistence: - size: 10Gi -buckets: - - name: demo - policy: public -resources: - requests: - cpu: 1 - memory: 2Gi -service: - type: NodePort - nodePort: null -consoleService: - type: NodePort - nodePort: null -tls: - enabled: true -extraVolumes: - # Request a TLS certificate from the secret-operator - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory -extraVolumeMounts: - # Mount the certificate generated by the secret-operator - - name: tls - mountPath: /etc/minio/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /etc/minio/certs - name: certs - -customCommandJob: - extraVolumes: - # Request a TLS certificate from the secret-operator - - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: |- - service=minio - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) - - name: certs - emptyDir: - sizeLimit: 5Mi - medium: Memory - # WARNING: this is currently only used by the custom-scripts job container. Other containers do not mount these. - extraVolumeMounts: - # Mount the certificate generated by the secret-operator - - name: tls - mountPath: /etc/minio/mc/original_certs - # On startup, we will rename the certs and move them here: - - mountPath: /etc/minio/mc/certs/CAs - name: certs diff --git a/stacks/_templates/minio.yaml b/stacks/_templates/minio.yaml deleted file mode 100644 index 135558ad..00000000 --- a/stacks/_templates/minio.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -releaseName: minio -name: minio -repo: - name: minio - url: https://charts.min.io/ -version: 5.4.0 # appVersion: RELEASE.2024-12-18T13-15-44Z -options: - additionalLabels: - stackable.tech/vendor: Stackable - stackable.tech/stack: "{{ STACK }}" - # The following is a Jinja2 template directive, not a regular comment. - # The `#` prefix is required to prevent YAML parsing errors, but Jinja2 still processes it. - # {% if DEMO is defined %} - stackable.tech/demo: "{{ DEMO }}" - # {% endif %} - podLabels: - stackable.tech/vendor: Stackable - stackable.tech/stack: "{{ STACK }}" - # The following is a Jinja2 template directive, not a regular comment. - # The `#` prefix is required to prevent YAML parsing errors, but Jinja2 still processes it. - # {% if DEMO is defined %} - stackable.tech/demo: "{{ DEMO }}" - # {% endif %} - rootUser: admin - rootPassword: "{{ minioAdminPassword }}" - mode: standalone - persistence: - size: 10Gi - buckets: - - name: demo - policy: public - resources: - requests: - cpu: 1 - memory: 2Gi - service: - type: NodePort - nodePort: null - consoleService: - type: NodePort - nodePort: null diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 850c1a3d..e1474c5a 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -89,14 +89,21 @@ spec: - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: minio-s3-credentials + name: s3-credentials key: secretKey - name: AWS_ACCESS_KEY_ID - value: admin + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + # boto3 has no region here otherwise, and Garage rejects requests + # signed for a different region than the one it is configured with. + - name: AWS_DEFAULT_REGION + value: region-1 - name: AIRFLOW_CONN_KAFKA_CONN value: '{"conn_type": "kafka", "extra": {"bootstrap.servers": "kafka-broker-default-0-listener-broker.$(NAMESPACE).svc.cluster.local:9093", "security.protocol": "SSL", "ssl.ca.location": "/stackable/tls-pem/ca.crt", "group.id": "airflow_group", "auto.offset.reset": "latest"}}' - name: AIRFLOW_CONN_S3_CONN - value: '{"conn_type": "aws", "extra": {"endpoint_url": "https://minio.$(NAMESPACE).svc.cluster.local:9000", "verify": "/stackable/tls-pem/ca.crt"}}' + value: '{"conn_type": "aws", "extra": {"endpoint_url": "https://garage.$(NAMESPACE).svc.cluster.local:9000", "verify": "/stackable/tls-pem/ca.crt"}}' roleGroups: default: replicas: 1 @@ -118,10 +125,15 @@ spec: - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: minio-s3-credentials + name: s3-credentials key: secretKey - name: AWS_ACCESS_KEY_ID - value: admin + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + - name: AWS_DEFAULT_REGION + value: region-1 schedulers: envOverrides: *envOverrides podOverrides: *podOverrides diff --git a/stacks/airflow/hive-metastores.yaml b/stacks/airflow/hive-metastores.yaml index 4870d3da..37ee0c62 100644 --- a/stacks/airflow/hive-metastores.yaml +++ b/stacks/airflow/hive-metastores.yaml @@ -13,7 +13,7 @@ spec: database: hive credentialsSecretName: hive-postgres-credentials s3: - reference: minio + reference: garage metastore: roleGroups: default: diff --git a/stacks/airflow/minio.yaml b/stacks/airflow/minio.yaml deleted file mode 100644 index b34cfe3a..00000000 --- a/stacks/airflow/minio.yaml +++ /dev/null @@ -1,715 +0,0 @@ ---- -# Source: minio/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "minio-sa" ---- -# Source: minio/templates/secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- -# Source: minio/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -data: - initialize: |- - #!/bin/sh - set -e # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 - LIMIT=29 # Allow 30 attempts - set -e # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) - SECRET=$(cat /config/rootPassword) - set +e # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" - $MC_COMMAND - STATUS=$? - until [ $STATUS = 0 ]; do - ATTEMPTS=$(expr $ATTEMPTS + 1) - echo \"Failed attempts: $ATTEMPTS\" - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 - fi - sleep 2 # 1 second intervals between attempts - $MC_COMMAND - STATUS=$? - done - set -e # reset `e` as active - return 0 - } - - # checkBucketExists ($bucket) - # Check if the bucket exists, by using the exit code of `mc ls` - checkBucketExists() { - BUCKET=$1 - CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) - return $? - } - - # createBucket ($bucket, $policy, $purge) - # Ensure bucket exists, purging if asked to - createBucket() { - BUCKET=$1 - POLICY=$2 - PURGE=$3 - VERSIONING=$4 - OBJECTLOCKING=$5 - - # Purge the bucket, if set & exists - # Since PURGE is user input, check explicitly for `true` - if [ $PURGE = true ]; then - if checkBucketExists $BUCKET; then - echo "Purging bucket '$BUCKET'." - set +e # don't exit if this fails - ${MC} rm -r --force myminio/$BUCKET - set -e # reset `e` as active - else - echo "Bucket '$BUCKET' does not exist, skipping purge." - fi - fi - - # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) - if ! checkBucketExists $BUCKET; then - if [ ! -z $OBJECTLOCKING ]; then - if [ $OBJECTLOCKING = true ]; then - echo "Creating bucket with OBJECTLOCKING '$BUCKET'" - ${MC} mb --with-lock myminio/$BUCKET - elif [ $OBJECTLOCKING = false ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - fi - elif [ -z $OBJECTLOCKING ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - else - echo "Bucket '$BUCKET' already exists." - fi - fi - - # set versioning for bucket if objectlocking is disabled or not set - if [ $OBJECTLOCKING = false ]; then - if [ ! -z $VERSIONING ]; then - if [ $VERSIONING = true ]; then - echo "Enabling versioning for '$BUCKET'" - ${MC} version enable myminio/$BUCKET - elif [ $VERSIONING = false ]; then - echo "Suspending versioning for '$BUCKET'" - ${MC} version suspend myminio/$BUCKET - fi - fi - else - echo "Bucket '$BUCKET' versioning unchanged." - fi - - # At this point, the bucket should exist, skip checking for existence - # Set policy on the bucket - echo "Setting policy of bucket '$BUCKET' to '$POLICY'." - ${MC} anonymous set $POLICY myminio/$BUCKET - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - # Create the buckets - createBucket demo "public" false false false - createBucket airflow "public" false false false - - add-user: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkUserExists () - # Check if the user exists, by using the exit code of `mc admin user info` - checkUserExists() { - CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createUser ($policy) - createUser() { - POLICY=$1 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the user if it does not exist - if ! checkUserExists ; then - echo "Creating user '$USER'" - cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio - else - echo "User '$USER' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - - # set policy for user - if [ ! -z $POLICY -a $POLICY != " " ] ; then - echo "Adding policy '$POLICY' for '$USER'" - set +e ; # policy already attach errors out, allow it. - ${MC} admin policy attach myminio $POLICY --user=$USER - set -e - else - echo "User '$USER' has no policy attached." - fi - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - # Create the users - echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP - echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP - createUser consoleAdmin - - add-policy: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkPolicyExists ($policy) - # Check if the policy exists, by using the exit code of `mc admin policy info` - checkPolicyExists() { - POLICY=$1 - CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) - return $? - } - - # createPolicy($name, $filename) - createPolicy () { - NAME=$1 - FILENAME=$2 - - # Create the name if it does not exist - echo "Checking policy: $NAME (in /config/$FILENAME.json)" - if ! checkPolicyExists $NAME ; then - echo "Creating policy '$NAME'" - else - echo "Policy '$NAME' already exists." - fi - ${MC} admin policy create myminio $NAME /config/$FILENAME.json - - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - add-svcacct: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 2 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkSvcacctExists () - # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` - checkSvcacctExists() { - CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createSvcacct ($user) - createSvcacct () { - USER=$1 - FILENAME=$2 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the svcacct if it does not exist - if ! checkSvcacctExists ; then - echo "Creating svcacct '$SVCACCT'" - # Check if policy file is define - if [ -z $FILENAME ]; then - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER - else - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER - fi - else - echo "Svcacct '$SVCACCT' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - - custom-command: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # runCommand ($@) - # Run custom mc command - runCommand() { - ${MC} "$@" - return $? - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme ---- -# Source: minio/templates/pvc.yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: "10Gi" ---- -# Source: minio/templates/console-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio-console - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: minio - release: minio ---- -# Source: minio/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - monitoring: "true" -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -# Source: minio/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable -spec: - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 - replicas: 1 - selector: - matchLabels: - app: minio - release: minio - template: - metadata: - name: minio - labels: - app: minio - release: minio - stackable.tech/vendor: Stackable - annotations: - checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 - checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 - spec: - securityContext: - - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - runAsGroup: 1000 - runAsUser: 1000 - - serviceAccountName: minio-sa - containers: - - name: minio - image: "docker.io/pgsty/minio:RELEASE.2026-08-04T00-00-00Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # minio requires the TLS key pair to be specially named - # mkdir -p /etc/minio/certs - cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt - cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key - - /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 - volumeMounts: - - name: minio-user - mountPath: "/tmp/credentials" - readOnly: true - - name: export - mountPath: /export - - mountPath: /etc/minio/original_certs - name: tls - - mountPath: /etc/minio/certs - name: certs - ports: - - name: https - containerPort: 9000 - - name: https-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: rootUser - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: rootPassword - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - resources: - requests: - cpu: 1 - memory: 2Gi - securityContext: - readOnlyRootFilesystem: false - volumes: - - name: export - persistentVolumeClaim: - claimName: minio - - name: minio-user - secret: - secretName: minio - - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs ---- -# Source: minio/templates/post-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job - labels: - app: minio-post-job - chart: minio-5.4.0 - release: minio - heritage: Helm - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation -spec: - backoffLimit: 50 - template: - metadata: - labels: - app: minio-job - release: minio - stackable.tech/vendor: Stackable - spec: - restartPolicy: OnFailure - volumes: - - name: etc-path - emptyDir: {} - - name: tmp - emptyDir: {} - - name: minio-configuration - projected: - sources: - - configMap: - name: minio - - secret: - name: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - serviceAccountName: minio-sa - containers: - - name: minio-make-bucket - image: "docker.io/pgsty/mc:RELEASE.2026-09-16T00-00-00Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/initialize - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi - - name: minio-make-user - image: "docker.io/pgsty/mc:RELEASE.2026-09-16T00-00-00Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/add-user - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi diff --git a/stacks/airflow/s3-connection.yaml b/stacks/airflow/s3-connection.yaml index 610e6541..6321d7f4 100644 --- a/stacks/airflow/s3-connection.yaml +++ b/stacks/airflow/s3-connection.yaml @@ -2,13 +2,18 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - host: minio.{{ NAMESPACE }}.svc.cluster.local + host: garage.{{ NAMESPACE }}.svc.cluster.local port: 9000 + # Must match s3_region in stacks/_templates/garage.yaml: Garage rejects + # requests signed for a different region. Deliberately not an AWS region: + # the Hadoop S3 implementation falls back to us-east-2 when none is set. + region: + name: region-1 accessStyle: Path credentials: - secretClass: minio-s3-credentials + secretClass: s3-credentials tls: verification: server: @@ -18,7 +23,7 @@ spec: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: minio-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -28,9 +33,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: minio-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: - accessKey: admin - secretKey: {{ minioAdminPassword }} + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/stacks/airflow/trino.yaml b/stacks/airflow/trino.yaml index d54dea20..6dda6d89 100644 --- a/stacks/airflow/trino.yaml +++ b/stacks/airflow/trino.yaml @@ -87,7 +87,7 @@ spec: metastore: configMap: hive-iceberg s3: - reference: minio + reference: garage --- apiVersion: v1 kind: ConfigMap diff --git a/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml b/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml index d01e305a..8d325b2f 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml @@ -13,7 +13,7 @@ spec: database: hive credentialsSecretName: hive-postgres-credentials s3: - reference: minio + reference: garage metastore: roleGroups: default: @@ -34,7 +34,7 @@ spec: database: hive credentialsSecretName: hive-postgres-credentials s3: - reference: minio + reference: garage metastore: roleGroups: default: diff --git a/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml b/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml index 952d2898..db576c30 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml @@ -32,7 +32,7 @@ spec: configOverrides: broker.properties: num.partitions: "27" - log.segment.bytes: "50000000" # 0.5GB + log.segment.bytes: "50000000" # 50 MB log.retention.bytes: "2000000000" # 2 GB. Should keep between 2.0 and 2.5GB --- apiVersion: authentication.stackable.tech/v1alpha1 diff --git a/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml b/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml index 610e6541..6321d7f4 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml @@ -2,13 +2,18 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - host: minio.{{ NAMESPACE }}.svc.cluster.local + host: garage.{{ NAMESPACE }}.svc.cluster.local port: 9000 + # Must match s3_region in stacks/_templates/garage.yaml: Garage rejects + # requests signed for a different region. Deliberately not an AWS region: + # the Hadoop S3 implementation falls back to us-east-2 when none is set. + region: + name: region-1 accessStyle: Path credentials: - secretClass: minio-s3-credentials + secretClass: s3-credentials tls: verification: server: @@ -18,7 +23,7 @@ spec: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: minio-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -28,9 +33,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: minio-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: - accessKey: admin - secretKey: {{ minioAdminPassword }} + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml index 23f04a93..9422f4f1 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml @@ -13,7 +13,7 @@ spec: metastore: configMap: hive s3: - reference: minio + reference: garage --- apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCatalog @@ -27,7 +27,7 @@ spec: metastore: configMap: hive-iceberg s3: - reference: minio + reference: garage --- apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCatalog diff --git a/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml b/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml index 7d5ca771..f54f188e 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub-native-auth.yaml @@ -37,6 +37,11 @@ options: singleuser: cmd: null serviceAccountName: hub + cloudMetadata: + # Disable, otherwise we get Back-off restarting failed container block-cloud-metadata in pod: + # block-cloud-metadata iptables v1.8.9 (legacy): can't initialize iptables table `filter': Table does not exist (do you need to insmod?) + # I believe this breaks in Kernel 7.x. + blockWithIptables: false networkPolicy: enabled: false extraLabels: diff --git a/stacks/jupyterhub-keycloak/jupyterhub.yaml b/stacks/jupyterhub-keycloak/jupyterhub.yaml index 62382588..ce176e0f 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub.yaml @@ -46,6 +46,7 @@ options: metadata: annotations: secrets.stackable.tech/class: tls + secrets.stackable.tech/provision-parts: public spec: storageClassName: secrets.stackable.tech accessModes: @@ -145,6 +146,11 @@ options: singleuser: cmd: null serviceAccountName: spark + cloudMetadata: + # Disable, otherwise we get Back-off restarting failed container block-cloud-metadata in pod: + # block-cloud-metadata iptables v1.8.9 (legacy): can't initialize iptables table `filter': Table does not exist (do you need to insmod?) + # I believe this breaks in Kernel 7.x. + blockWithIptables: false networkPolicy: enabled: false extraLabels: @@ -155,6 +161,15 @@ options: # {% if DEMO is defined %} stackable.tech/demo: "{{ DEMO }}" # {% endif %} + extraEnv: + # Needed because the Notebook passes the namespace to Spark as + # spark.kubernetes.namespace. Without it the driver looks for its own Pod + # in "default" and the request is rejected by RBAC. + # We cannot use the downward API because some python code uses it and expects scalar values. + # We also MUST use a dict, otherwise hub shows the following error: + # traitlets.traitlets.TraitError: The 'environment' trait of a KubeSpawner instance expected a dict, not the list [{'name': 'NAMESPACE', 'value': 'my-namespace'}]. + # A helper in the chart formats these properly for any container env vars in Pods. + NAMESPACE: "{{ NAMESPACE }}" initContainers: - name: download-notebook image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev @@ -171,6 +186,7 @@ options: metadata: annotations: secrets.stackable.tech/class: tls + secrets.stackable.tech/provision-parts: public spec: storageClassName: secrets.stackable.tech accessModes: @@ -178,17 +194,17 @@ options: resources: requests: storage: "1" - - name: minio-s3-credentials + - name: s3-credentials secret: - secretName: minio-s3-credentials + secretName: s3-credentials - name: notebook emptyDir: sizeLimit: 500Mi extraVolumeMounts: - name: tls-ca-cert mountPath: /stackable/secrets/tls-ca-cert - - name: minio-s3-credentials - mountPath: /minio-s3-credentials + - name: s3-credentials + mountPath: /s3-credentials - name: notebook mountPath: /home/jovyan/notebook profileList: @@ -213,7 +229,7 @@ options: memory: display_name: Memory choices: -# {% for memory in ["1","2","4","8","16","32","64","128"] %} +# {% for memory in ["4","8","16","32","64","128"] %} "{{memory}} GB": display_name: "{{memory}} GB" kubespawner_override: diff --git a/stacks/jupyterhub-keycloak/process-s3.ipynb b/stacks/jupyterhub-keycloak/process-s3.ipynb index c28d3b7c..3e151a88 100644 --- a/stacks/jupyterhub-keycloak/process-s3.ipynb +++ b/stacks/jupyterhub-keycloak/process-s3.ipynb @@ -4,10 +4,7 @@ "cell_type": "markdown", "id": "f6515406-dc52-4a2b-9ae8-99fff7773146", "metadata": {}, - "source": [ - "## Preliminaries\n", - "We can first output some versions that are running and read the minio credentials from the secret that has been mounted." - ] + "source": "## Preliminaries\nWe can first output some versions that are running and read the S3 credentials from the secret that has been mounted." }, { "cell_type": "code", @@ -27,16 +24,7 @@ "id": "bd941fee", "metadata": {}, "outputs": [], - "source": [ - "import os\n", - "\n", - "# get minio credentials\n", - "with open(\"/minio-s3-credentials/accessKey\", \"r\") as f:\n", - " minio_user = f.read().strip()\n", - "\n", - "with open(\"/minio-s3-credentials/secretKey\", \"r\") as f:\n", - " minio_pwd = f.read().strip()" - ] + "source": "import os\n\n# get S3 credentials\nwith open(\"/s3-credentials/accessKey\", \"r\") as f:\n s3_user = f.read().strip()\n\nwith open(\"/s3-credentials/secretKey\", \"r\") as f:\n s3_pwd = f.read().strip()" }, { "cell_type": "markdown", @@ -63,38 +51,7 @@ "id": "606363ba-0c97-4156-af1c-c8ad54745cfb", "metadata": {}, "outputs": [], - "source": [ - "from pyspark.sql import SparkSession\n", - "\n", - "NAMESPACE = os.environ.get(\"NAMESPACE\", \"default\")\n", - "POD_NAME = os.environ.get(\"HOSTNAME\", f\"jupyter-{os.environ.get('USER', 'default')}-{NAMESPACE}\")\n", - "\n", - "EXECUTOR_IMAGE = \"oci.stackable.tech/demos/spark:3.5.2-python311\" \n", - "\n", - "spark = (\n", - " SparkSession.builder\n", - " .master(f\"k8s://https://{os.environ['KUBERNETES_SERVICE_HOST']}:{os.environ['KUBERNETES_SERVICE_PORT']}\")\n", - " .appName(f\"process-s3-{POD_NAME}\")\n", - " .config(\"spark.kubernetes.container.image\", EXECUTOR_IMAGE)\n", - " .config(\"spark.kubernetes.container.image.pullPolicy\", \"IfNotPresent\")\n", - " .config(\"spark.kubernetes.namespace\", NAMESPACE)\n", - " .config(\"spark.kubernetes.authenticate.driver.serviceAccountName\", \"spark\")\n", - " .config(\"spark.kubernetes.authenticate.executor.serviceAccountName\", \"spark\")\n", - " .config(\"spark.driver.port\", \"2222\")\n", - " .config(\"spark.driver.blockManager.port\", \"7777\")\n", - " .config(\"spark.executor.instances\", \"1\")\n", - " .config(\"spark.executor.memory\", \"1g\")\n", - " .config(\"spark.executor.cores\", \"1\")\n", - " .config(\"spark.hadoop.fs.s3a.endpoint\", \"http://minio:9000/\")\n", - " .config(\"spark.hadoop.fs.s3a.path.style.access\", \"true\")\n", - " .config(\"spark.hadoop.fs.s3a.access.key\", minio_user)\n", - " .config(\"spark.hadoop.fs.s3a.secret.key\", minio_pwd)\n", - " .config(\"spark.hadoop.fs.s3a.aws.credentials.provider\", \"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider\")\n", - " .config(\"spark.jars.packages\", \"org.apache.hadoop:hadoop-client-api:3.3.4,org.apache.hadoop:hadoop-client-runtime:3.3.4,org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-common:3.3.4,com.amazonaws:aws-java-sdk-bundle:1.12.162\")\n", - " .config(\"spark.kubernetes.driver.pod.name\", POD_NAME)\n", - " .getOrCreate()\n", - ")" - ] + "source": "from pyspark.sql import SparkSession\n\nNAMESPACE = os.environ.get(\"NAMESPACE\", \"default\")\nPOD_NAME = os.environ.get(\"HOSTNAME\", f\"jupyter-{os.environ.get('USER', 'default')}-{NAMESPACE}\")\n\nEXECUTOR_IMAGE = \"oci.stackable.tech/demos/spark:3.5.2-python311\" \n\nspark = (\n SparkSession.builder\n .master(f\"k8s://https://{os.environ['KUBERNETES_SERVICE_HOST']}:{os.environ['KUBERNETES_SERVICE_PORT']}\")\n .appName(f\"process-s3-{POD_NAME}\")\n .config(\"spark.kubernetes.container.image\", EXECUTOR_IMAGE)\n .config(\"spark.kubernetes.container.image.pullPolicy\", \"IfNotPresent\")\n .config(\"spark.kubernetes.namespace\", NAMESPACE)\n .config(\"spark.kubernetes.authenticate.driver.serviceAccountName\", \"spark\")\n .config(\"spark.kubernetes.authenticate.executor.serviceAccountName\", \"spark\")\n .config(\"spark.driver.port\", \"2222\")\n .config(\"spark.driver.blockManager.port\", \"7777\")\n .config(\"spark.executor.instances\", \"1\")\n .config(\"spark.executor.memory\", \"1g\")\n .config(\"spark.executor.cores\", \"1\")\n .config(\"spark.hadoop.fs.s3a.endpoint\", \"http://garage-plaintext:3900/\")\n # Without this the Hadoop S3 implementation signs for us-east-2, and the\n # request is rejected as being signed for the wrong region.\n .config(\"spark.hadoop.fs.s3a.endpoint.region\", \"region-1\")\n .config(\"spark.hadoop.fs.s3a.path.style.access\", \"true\")\n .config(\"spark.hadoop.fs.s3a.access.key\", s3_user)\n .config(\"spark.hadoop.fs.s3a.secret.key\", s3_pwd)\n .config(\"spark.hadoop.fs.s3a.aws.credentials.provider\", \"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider\")\n .config(\"spark.jars.packages\", \"org.apache.hadoop:hadoop-client-api:3.3.4,org.apache.hadoop:hadoop-client-runtime:3.3.4,org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-common:3.3.4,com.amazonaws:aws-java-sdk-bundle:1.12.162\")\n .config(\"spark.kubernetes.driver.pod.name\", POD_NAME)\n .getOrCreate()\n)" }, { "cell_type": "markdown", @@ -131,15 +88,7 @@ "id": "9f8479cb-f216-4a8f-b9db-6da17ffebaa9", "metadata": {}, "outputs": [], - "source": [ - "# Manual S3 file check via pyarrow.fs\n", - "import pyarrow.fs as fs\n", - "\n", - "s3 = fs.S3FileSystem(endpoint_override=\"http://minio:9000/\", access_key=minio_user, secret_key=minio_pwd, scheme=\"http\")\n", - "files = s3.get_file_info(fs.FileSelector(\"demo/gas-sensor/raw/\", recursive=True))\n", - "for f in files:\n", - " print(\"Found file:\", f.path)" - ] + "source": "# Manual S3 file check via pyarrow.fs\nimport pyarrow.fs as fs\n\ns3 = fs.S3FileSystem(endpoint_override=\"http://garage-plaintext:3900/\", access_key=s3_user, secret_key=s3_pwd, region=\"region-1\", scheme=\"http\")\nfiles = s3.get_file_info(fs.FileSelector(\"demo/gas-sensor/raw/\", recursive=True))\nfor f in files:\n print(\"Found file:\", f.path)" }, { "cell_type": "markdown", @@ -261,4 +210,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/stacks/jupyterhub-keycloak/s3-connection.yaml b/stacks/jupyterhub-keycloak/s3-connection.yaml index 7e026539..6321d7f4 100644 --- a/stacks/jupyterhub-keycloak/s3-connection.yaml +++ b/stacks/jupyterhub-keycloak/s3-connection.yaml @@ -2,18 +2,28 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - host: minio + host: garage.{{ NAMESPACE }}.svc.cluster.local port: 9000 + # Must match s3_region in stacks/_templates/garage.yaml: Garage rejects + # requests signed for a different region. Deliberately not an AWS region: + # the Hadoop S3 implementation falls back to us-east-2 when none is set. + region: + name: region-1 accessStyle: Path credentials: - secretClass: minio-s3-credentials + secretClass: s3-credentials + tls: + verification: + server: + caCert: + secretClass: tls --- apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: minio-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -23,9 +33,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: minio-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: - accessKey: admin - secretKey: {{ minioAdminPassword }} + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/stacks/monitoring/grafana-dashboards.yaml b/stacks/monitoring/grafana-dashboards.yaml index a193c208..78393adc 100644 --- a/stacks/monitoring/grafana-dashboards.yaml +++ b/stacks/monitoring/grafana-dashboards.yaml @@ -7463,6 +7463,1125 @@ data: "uid": "TgmJnqnnk", "version": 1 } + garage.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Garage S3 object store. Counters such as request, error and block I/O rates are only created once the corresponding operation has happened at least once, so those panels read as No data on a freshly installed stack.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Whether all storage nodes are connected.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Unhealthy" + }, + "1": { + "color": "green", + "index": 0, + "text": "Healthy" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_healthy{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "healthy", + "range": false, + "refId": "A" + } + ], + "title": "Cluster Health", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Whether all requests can be served, even if some storage nodes are disconnected.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Degraded" + }, + "1": { + "color": "green", + "index": 0, + "text": "Available" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_available{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "available", + "range": false, + "refId": "A" + } + ], + "title": "Cluster Available", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Nodes currently connected, against nodes seen at least once.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_connected_nodes{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "connected", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_known_nodes{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "known", + "range": false, + "refId": "B" + } + ], + "title": "Connected Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Storage nodes connected, against storage nodes declared in the layout.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_storage_nodes_ok{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "ok", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_storage_nodes{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "declared", + "range": false, + "refId": "B" + } + ], + "title": "Storage Nodes OK", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Partitions with a quorum of connected nodes, against the total in the layout.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 4 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_partitions_quorum{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "quorum", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(cluster_partitions{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "total", + "range": false, + "refId": "B" + } + ], + "title": "Partitions with Quorum", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of buckets, from the size of Garage's internal bucket table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 4 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(table_size{namespace=~\"$namespace\", table_name=\"bucket_v2\"})", + "instant": true, + "legendFormat": "buckets", + "range": false, + "refId": "A" + } + ], + "title": "Buckets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of objects, from the size of Garage's internal object table.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 4 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(table_size{namespace=~\"$namespace\", table_name=\"object\"})", + "instant": true, + "legendFormat": "objects", + "range": false, + "refId": "A" + } + ], + "title": "Objects", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Configured number of copies of each object.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 4 + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(garage_replication_factor{namespace=~\"$namespace\"})", + "instant": true, + "legendFormat": "factor", + "range": false, + "refId": "A" + } + ], + "title": "Replication Factor", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Data volume only. The metadata volume is reported separately and is often the same filesystem.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(garage_local_disk_total{namespace=~\"$namespace\", volume=\"data\"}) - sum(garage_local_disk_avail{namespace=~\"$namespace\", volume=\"data\"})", + "instant": false, + "legendFormat": "used", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(garage_local_disk_avail{namespace=~\"$namespace\", volume=\"data\"})", + "instant": false, + "legendFormat": "free", + "range": true, + "refId": "B" + } + ], + "title": "Disk Capacity", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Data block throughput to and from disk.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(block_bytes_read{namespace=~\"$namespace\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "read", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(block_bytes_written{namespace=~\"$namespace\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "written", + "range": true, + "refId": "B" + } + ], + "title": "Block I/O Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "S3 API calls per second, by endpoint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (api_endpoint) (rate(api_s3_request_counter{namespace=~\"$namespace\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{api_endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "S3 API Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "S3 API calls resulting in an error, by endpoint and HTTP status.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (api_endpoint, status_code) (rate(api_s3_error_counter{namespace=~\"$namespace\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{api_endpoint}} {{status_code}}", + "range": true, + "refId": "A" + } + ], + "title": "S3 API Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "95th percentile S3 API call duration, by endpoint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, api_endpoint) (rate(api_s3_request_duration_bucket{namespace=~\"$namespace\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{api_endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "S3 API Latency (p95)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(cluster_healthy, namespace)", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "query": "label_values(cluster_healthy, namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Garage", + "uid": "garage-overview", + "version": 1 + } hdfs.json: | { "annotations": { diff --git a/stacks/monitoring/prometheus-service-monitors.yaml b/stacks/monitoring/prometheus-service-monitors.yaml index 0b1acdd2..e1c752e5 100644 --- a/stacks/monitoring/prometheus-service-monitors.yaml +++ b/stacks/monitoring/prometheus-service-monitors.yaml @@ -147,3 +147,23 @@ spec: # Prevent "tls: failed to verify certificate: x509: cannot validate certificate for 100.96.234.154 because it doesn't contain any IP SANs" tlsConfig: insecureSkipVerify: true +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: stackable-garage + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + namespaceSelector: + any: true + selector: + matchLabels: + app: garage + endpoints: + # Garage terminates no TLS itself. The TLS endpoint in front of it is the S3 + # API on another Service; this is the admin API, which is HTTP only. + - scheme: http + port: metrics + path: /metrics diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml index e456ba25..76523c4d 100644 --- a/stacks/nifi-kafka-druid-superset-s3/druid.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/druid.yaml @@ -20,8 +20,15 @@ spec: bucketName: demo connection: inline: - host: minio.{{ NAMESPACE }}.svc.cluster.local + host: garage.{{ NAMESPACE }}.svc.cluster.local port: 9000 + # Must match s3_region in stacks/_templates/garage.yaml: Garage + # rejects requests signed for a different region. + # Only honoured for Druid >= 37.0.0, which uses the AWS SDK v2. + # Older versions use SDK v1, which ignores the region when an + # endpoint is set. + region: + name: region-1 accessStyle: Path credentials: secretClass: druid-s3-credentials @@ -108,8 +115,8 @@ metadata: labels: secrets.stackable.tech/class: druid-s3-credentials stringData: - accessKey: admin - secretKey: {{ minioAdminPassword }} + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef --- apiVersion: v1 kind: Secret diff --git a/stacks/spark-trino-superset-s3/hive-metastore.yaml b/stacks/spark-trino-superset-s3/hive-metastore.yaml index d01e305a..8d325b2f 100644 --- a/stacks/spark-trino-superset-s3/hive-metastore.yaml +++ b/stacks/spark-trino-superset-s3/hive-metastore.yaml @@ -13,7 +13,7 @@ spec: database: hive credentialsSecretName: hive-postgres-credentials s3: - reference: minio + reference: garage metastore: roleGroups: default: @@ -34,7 +34,7 @@ spec: database: hive credentialsSecretName: hive-postgres-credentials s3: - reference: minio + reference: garage metastore: roleGroups: default: diff --git a/stacks/spark-trino-superset-s3/s3-connection.yaml b/stacks/spark-trino-superset-s3/s3-connection.yaml index e52f7183..6321d7f4 100644 --- a/stacks/spark-trino-superset-s3/s3-connection.yaml +++ b/stacks/spark-trino-superset-s3/s3-connection.yaml @@ -2,14 +2,18 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - # Hostname must match secret op's certificate SAN - host: minio.{{ NAMESPACE }}.svc.cluster.local + host: garage.{{ NAMESPACE }}.svc.cluster.local port: 9000 + # Must match s3_region in stacks/_templates/garage.yaml: Garage rejects + # requests signed for a different region. Deliberately not an AWS region: + # the Hadoop S3 implementation falls back to us-east-2 when none is set. + region: + name: region-1 accessStyle: Path credentials: - secretClass: minio-s3-credentials + secretClass: s3-credentials tls: verification: server: @@ -19,7 +23,7 @@ spec: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: minio-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -29,9 +33,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: minio-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: - accessKey: admin - secretKey: {{ minioAdminPassword }} + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/stacks/spark-trino-superset-s3/trino-prediction-catalog.yaml b/stacks/spark-trino-superset-s3/trino-prediction-catalog.yaml index 0410fa17..e724ba93 100644 --- a/stacks/spark-trino-superset-s3/trino-prediction-catalog.yaml +++ b/stacks/spark-trino-superset-s3/trino-prediction-catalog.yaml @@ -11,4 +11,4 @@ spec: metastore: configMap: hive-iceberg s3: - reference: minio + reference: garage diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index e7039423..42776776 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -148,10 +148,11 @@ stacks: - trino - opa - iceberg - - minio + - garage - s3 - airflow manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-airflow.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/kafka.yaml @@ -162,7 +163,6 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/rbac.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/opa-rules.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/opa.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/minio.yaml supportedNamespaces: [] resourceRequests: cpu: 3400m @@ -172,9 +172,12 @@ stacks: - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin - - name: minioAdminPassword - description: Password of the MinIO admin user - default: adminadmin + - name: garageBuckets + description: Space separated list of S3 buckets to create in Garage + default: demo airflow + - name: garageStorageSize + description: Size of the PersistentVolumeClaim backing the Garage object data + default: 10Gi - name: airflowAdminPassword description: Password of the Airflow admin user default: adminadmin @@ -203,10 +206,10 @@ stacks: - superset - kafka - nifi - - minio + - garage - s3 manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-distributed-tls/rendered-chart.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml @@ -220,7 +223,8 @@ stacks: resourceRequests: cpu: "71" memory: 160Gi - pvc: 1Ti + # 250Gi Kafka (5 brokers x 50Gi), 50Gi Garage, 34Gi NiFi, 24Gi PostgreSQL + pvc: 400Gi parameters: - name: trinoAdminPassword description: Password of the Trino admin user @@ -229,11 +233,14 @@ stacks: description: Password of the Superset admin user default: adminadmin - name: nifiAdminPassword - description: Password of the MinIO admin user - default: adminadmin - - name: minioAdminPassword - description: Password of the MinIO admin user - default: adminadmin + description: Password of the NiFi admin user + default: adminadmin + - name: garageBuckets + description: Space separated list of S3 buckets to create in Garage + default: staging lakehouse + - name: garageStorageSize + description: Size of the PersistentVolumeClaim backing the Garage object data + default: 50Gi hdfs-hbase: description: HBase cluster using HDFS as underlying storage stackableRelease: dev @@ -258,7 +265,7 @@ stacks: pvc: 21Gi parameters: [] nifi-kafka-druid-superset-s3: - description: Stack containing NiFi, Kafka, Druid, MinIO and Superset for data visualization + description: Stack containing NiFi, Kafka, Druid, Garage and Superset for data visualization stackableRelease: dev stackableOperators: - commons @@ -274,10 +281,10 @@ stacks: - kafka - druid - superset - - minio + - garage - s3 manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-tls/rendered-chart.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-druid.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/zookeeper.yaml @@ -297,11 +304,14 @@ stacks: - name: supersetAdminPassword description: Password of the Superset admin user default: adminadmin - - name: minioAdminPassword - description: Password of the MinIO admin user - default: adminadmin + - name: garageBuckets + description: Space separated list of S3 buckets to create in Garage + default: demo + - name: garageStorageSize + description: Size of the PersistentVolumeClaim backing the Garage object data + default: 10Gi spark-trino-superset-s3: - description: Stack containing MinIO, Trino and Superset for data visualization + description: Stack containing Garage, Trino and Superset for data visualization stackableRelease: dev stackableOperators: - commons @@ -315,10 +325,10 @@ stacks: labels: - trino - superset - - minio + - garage - s3 manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-tls/rendered-chart.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml @@ -341,11 +351,14 @@ stacks: - name: supersetAdminPassword description: Password of the Superset admin user default: adminadmin - - name: minioAdminPassword - description: Password of the MinIO admin user - default: adminadmin + - name: garageBuckets + description: Space separated list of S3 buckets to create in Garage + default: demo prediction + - name: garageStorageSize + description: Size of the PersistentVolumeClaim backing the Garage object data + default: 10Gi trino-superset-s3: - description: Stack containing MinIO, Trino and Superset for data visualization + description: Stack containing Garage, Trino and Superset for data visualization stackableRelease: dev stackableOperators: - commons @@ -358,10 +371,10 @@ stacks: labels: - trino - superset - - minio + - garage - s3 manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-tls/rendered-chart.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-superset-s3/s3-connection.yaml @@ -380,9 +393,12 @@ stacks: - name: supersetAdminPassword description: Password of the Superset admin user default: adminadmin - - name: minioAdminPassword - description: Password of the MinIO admin user - default: adminadmin + - name: garageBuckets + description: Space separated list of S3 buckets to create in Garage + default: demo + - name: garageStorageSize + description: Size of the PersistentVolumeClaim backing the Garage object data + default: 10Gi trino-iceberg: description: Stack containing Trino using Apache Iceberg as a S3 data lakehouse stackableRelease: dev @@ -397,10 +413,10 @@ stacks: - trino - opa - iceberg - - minio + - garage - s3 manifests: - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-distributed-small-tls/rendered-chart.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/s3-connection.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/trino-iceberg/hive-metastores.yaml @@ -409,14 +425,17 @@ stacks: resourceRequests: cpu: 6000m # Measured 5600m memory: 21Gi - pvc: 110Gi # 100Gi for MinIO + pvc: 60Gi # 50Gi for Garage parameters: - name: trinoAdminPassword description: Password of the Trino admin user default: adminadmin - - name: minioAdminPassword - description: Password of the MinIO admin user - default: adminadmin + - name: garageBuckets + description: Space separated list of S3 buckets to create in Garage + default: lakehouse + - name: garageStorageSize + description: Size of the PersistentVolumeClaim backing the Garage object data + default: 50Gi jupyterhub-pyspark-hdfs: description: Jupyterhub with PySpark and HDFS integration stackableRelease: dev @@ -621,7 +640,7 @@ stacks: memory: 9010Mi pvc: 24Gi manifests: - - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/keycloak-realm-config.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-keycloak/keycloak.yaml @@ -632,9 +651,12 @@ stacks: - name: keycloakAdminPassword description: Password of the Keycloak admin user default: adminadmin - - name: minioAdminPassword - description: Password of the MinIO admin user - default: adminadmin + - name: garageBuckets + description: Space separated list of S3 buckets to create in Garage + default: demo + - name: garageStorageSize + description: Size of the PersistentVolumeClaim backing the Garage object data + default: 10Gi - name: jupyterhubClientPassword description: Password of the JupyterHub client user default: jupyterhubjupyterhub diff --git a/stacks/trino-iceberg/hive-metastores.yaml b/stacks/trino-iceberg/hive-metastores.yaml index 4870d3da..37ee0c62 100644 --- a/stacks/trino-iceberg/hive-metastores.yaml +++ b/stacks/trino-iceberg/hive-metastores.yaml @@ -13,7 +13,7 @@ spec: database: hive credentialsSecretName: hive-postgres-credentials s3: - reference: minio + reference: garage metastore: roleGroups: default: diff --git a/stacks/trino-iceberg/s3-connection.yaml b/stacks/trino-iceberg/s3-connection.yaml index 610e6541..6321d7f4 100644 --- a/stacks/trino-iceberg/s3-connection.yaml +++ b/stacks/trino-iceberg/s3-connection.yaml @@ -2,13 +2,18 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - host: minio.{{ NAMESPACE }}.svc.cluster.local + host: garage.{{ NAMESPACE }}.svc.cluster.local port: 9000 + # Must match s3_region in stacks/_templates/garage.yaml: Garage rejects + # requests signed for a different region. Deliberately not an AWS region: + # the Hadoop S3 implementation falls back to us-east-2 when none is set. + region: + name: region-1 accessStyle: Path credentials: - secretClass: minio-s3-credentials + secretClass: s3-credentials tls: verification: server: @@ -18,7 +23,7 @@ spec: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: minio-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -28,9 +33,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: minio-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: - accessKey: admin - secretKey: {{ minioAdminPassword }} + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/stacks/trino-iceberg/trino.yaml b/stacks/trino-iceberg/trino.yaml index 490959e9..c80fc7e7 100644 --- a/stacks/trino-iceberg/trino.yaml +++ b/stacks/trino-iceberg/trino.yaml @@ -13,7 +13,7 @@ spec: metastore: configMap: hive-iceberg s3: - reference: minio + reference: garage --- apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCatalog diff --git a/stacks/trino-superset-s3/hive-metastore.yaml b/stacks/trino-superset-s3/hive-metastore.yaml index f804a57e..9b921809 100644 --- a/stacks/trino-superset-s3/hive-metastore.yaml +++ b/stacks/trino-superset-s3/hive-metastore.yaml @@ -13,7 +13,7 @@ spec: database: hive credentialsSecretName: hive-postgres-credentials s3: - reference: minio + reference: garage metastore: roleGroups: default: diff --git a/stacks/trino-superset-s3/s3-connection.yaml b/stacks/trino-superset-s3/s3-connection.yaml index 610e6541..6321d7f4 100644 --- a/stacks/trino-superset-s3/s3-connection.yaml +++ b/stacks/trino-superset-s3/s3-connection.yaml @@ -2,13 +2,18 @@ apiVersion: s3.stackable.tech/v1alpha1 kind: S3Connection metadata: - name: minio + name: garage spec: - host: minio.{{ NAMESPACE }}.svc.cluster.local + host: garage.{{ NAMESPACE }}.svc.cluster.local port: 9000 + # Must match s3_region in stacks/_templates/garage.yaml: Garage rejects + # requests signed for a different region. Deliberately not an AWS region: + # the Hadoop S3 implementation falls back to us-east-2 when none is set. + region: + name: region-1 accessStyle: Path credentials: - secretClass: minio-s3-credentials + secretClass: s3-credentials tls: verification: server: @@ -18,7 +23,7 @@ spec: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: minio-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -28,9 +33,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: minio-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: minio-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: - accessKey: admin - secretKey: {{ minioAdminPassword }} + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef diff --git a/stacks/trino-superset-s3/trino.yaml b/stacks/trino-superset-s3/trino.yaml index 5783b7a8..d6faaaf7 100644 --- a/stacks/trino-superset-s3/trino.yaml +++ b/stacks/trino-superset-s3/trino.yaml @@ -13,7 +13,7 @@ spec: metastore: configMap: hive s3: - reference: minio + reference: garage --- apiVersion: trino.stackable.tech/v1alpha1 kind: TrinoCluster