diff --git a/README.md b/README.md index b38db454..aabb737a 100644 --- a/README.md +++ b/README.md @@ -66,9 +66,10 @@ The cluster-deployment tools here include helm charts and ansible playbooks to s | ddclient | [![](https://img.shields.io/docker/v/instantlinux/ddclient?sort=date)](https://hub.docker.com/r/instantlinux/ddclient "Version badge") | Dynamic DNS client | | ez-ipupdate | [![](https://img.shields.io/docker/v/instantlinux/ez-ipupdate?sort=date)](https://hub.docker.com/r/instantlinux/ez-ipupdate "Version badge") | Dynamic DNS client | | fluent-bit | ** | central logging for Kubernetes | -| haproxy-keepalived | [![](https://img.shields.io/docker/v/instantlinux/haproxy-keepalived?sort=date)](https://hub.docker.com/r/instantlinux/haproxy-keepalived "Version badge") | load balancer | +| garage | ** | S3-compatible object storage | | grafana | ** | monitoring dashboard with prometheus-based alerting | | guacamole | ** | authenticated remote-desktop server | +| haproxy-keepalived | [![](https://img.shields.io/docker/v/instantlinux/haproxy-keepalived?sort=date)](https://hub.docker.com/r/instantlinux/haproxy-keepalived "Version badge") | load balancer | | headscale | ** | tailscale-compatible VPN control-plane | | mysqldump | [![](https://img.shields.io/docker/v/instantlinux/mysqldump?sort=date)](https://hub.docker.com/r/instantlinux/mysqldump "Version badge") | per-database alternative to xtrabackup | | nagios | [![](https://img.shields.io/docker/v/instantlinux/nagios?sort=date)](https://hub.docker.com/r/instantlinux/nagios "Version badge") | Nagios Core v4 for monitoring | diff --git a/ansible/roles/monitoring_agent/handlers/main.yml b/ansible/roles/monitoring_agent/handlers/main.yml index 77a228ef..2ab753b7 100644 --- a/ansible/roles/monitoring_agent/handlers/main.yml +++ b/ansible/roles/monitoring_agent/handlers/main.yml @@ -20,3 +20,6 @@ service: name: snmpd state: restarted + +- name: Reload apparmor + ansible.builtin.command: apparmor_parser -r /etc/apparmor.d/usr.sbin.rsyslogd diff --git a/ansible/roles/monitoring_agent/tasks/main.yml b/ansible/roles/monitoring_agent/tasks/main.yml index 44335bd4..57edcbc8 100644 --- a/ansible/roles/monitoring_agent/tasks/main.yml +++ b/ansible/roles/monitoring_agent/tasks/main.yml @@ -4,20 +4,20 @@ - import_tasks: packages.yml - name: Local nagios plugins path - file: + ansible.builtin.file: path: "{{ nagios.local_plugins_path }}" recurse: True state: directory - name: Install local NRPE plugins - copy: + ansible.builtin.copy: dest: "{{ nagios.local_plugins_path }}" force: no mode: 0755 src: plugins/ - name: Add NRPE to sudoers - template: + ansible.builtin.template: dest: /etc/sudoers.d/nrpe mode: 0440 src: nrpe_sudoer.j2 @@ -25,20 +25,20 @@ notify: Restart NRPE - name: Allow from nagios server hosts - lineinfile: + ansible.builtin.lineinfile: dest: "{{ nagios.nrpe_cfg }}" line: allowed_hosts={{ ','.join(nagios.allowed_hosts) }} regexp: "^allowed_hosts=.*" notify: Restart NRPE - name: Add local nrpe definitions - template: + ansible.builtin.template: dest: "{{ nagios.nrpe_path }}" src: nrpe_command.cfg.j2 notify: Restart NRPE - name: Set loglevel to INFO in snmpd - lineinfile: + ansible.builtin.lineinfile: dest: /etc/default/snmpd line: > SNMPDOPTS='-LS6d -Lf /dev/null -u snmp -g snmp @@ -47,37 +47,45 @@ notify: Restart snmpd - name: Enable NRPE service - service: + ansible.builtin.service: name: "{{ nagios.nrpe_service }}" enabled: yes state: started - name: Syslog host setup - template: + ansible.builtin.template: dest: /etc/rsyslog.d/custom.conf src: rsyslog-custom.conf.j2 notify: Restart rsyslogd - name: Move imuxsock load out of stock /etc/rsyslog.conf - lineinfile: + ansible.builtin.lineinfile: dest: /etc/rsyslog.conf line: "# module(load=\"imuxsock\")" regexp: "^module.load=\"imuxsock\".*" notify: Restart rsyslogd - name: Ensure logs are in UTC - template: + ansible.builtin.template: dest: /etc/systemd/system/rsyslog.service src: rsyslog.service.j2 notify: Restart rsyslogd +- name: Allow rsyslogd access to cmdline + ansible.builtin.copy: + content: | + # Managed by ansible + /proc/[0-9]*/cmdline r, + dest: /etc/apparmor.d/local/usr.sbin.rsyslogd + notify: Reload apparmor + - name: Put a leash on journald (default log files are huge) - template: + ansible.builtin.template: dest: /etc/systemd/journald.conf src: journald.conf.j2 notify: Restart journald - name: Define tolerated drive errors - template: + ansible.builtin.template: dest: "{{ nagios.local_plugins_path }}/tolerations.yaml" src: tolerations.j2 diff --git a/ansible/roles/network/defaults/main.yml b/ansible/roles/network/defaults/main.yml index 4e0bed5d..934b36c8 100644 --- a/ansible/roles/network/defaults/main.yml +++ b/ansible/roles/network/defaults/main.yml @@ -20,12 +20,15 @@ network_defaults: - to: default via: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['gateway'] }}" search: "{{ hostvars[inventory_hostname]['ansible_dns']['search'] }}" - tailscale: false + tailscale: + enabled: false + port: 41641 + telemetry: false cluster_ip: null vrrp_interface: null network_override: {} -network: "{{ network_defaults | combine(network_override) }}" +network: "{{ network_defaults | combine(network_override, recursive=True) }}" network_sysctl: net.ipv4.ip_nonlocal_bind: 1 diff --git a/ansible/roles/network/handlers/main.yml b/ansible/roles/network/handlers/main.yml index d470fbd8..e904614a 100644 --- a/ansible/roles/network/handlers/main.yml +++ b/ansible/roles/network/handlers/main.yml @@ -1,14 +1,20 @@ --- - name: Restart keepalived - service: + ansible.builtin.service: enabled: yes name: keepalived state: restarted - name: Netplan apply - command: netplan apply + ansible.builtin.command: netplan apply - name: Restart sshd - service: + ansible.builtin.service: name: ssh state: restarted + +- name: Restart tailscaled + ansible.builtin.service: + enabled: yes + name: tailscaled + state: restarted diff --git a/ansible/roles/network/tasks/main.yml b/ansible/roles/network/tasks/main.yml index 4045f490..56140629 100644 --- a/ansible/roles/network/tasks/main.yml +++ b/ansible/roles/network/tasks/main.yml @@ -10,7 +10,7 @@ when: ansible_facts['distribution_version'] >= '18.04' - import_tasks: tailscale.yml - when: network.tailscale + when: network.tailscale.enabled - import_tasks: kernel.yml - import_tasks: sshd.yml diff --git a/ansible/roles/network/tasks/tailscale.yml b/ansible/roles/network/tasks/tailscale.yml index 0b0cc30d..c6651101 100644 --- a/ansible/roles/network/tasks/tailscale.yml +++ b/ansible/roles/network/tasks/tailscale.yml @@ -17,6 +17,16 @@ update_cache: true state: present +- name: Disable telemetry to log.tailscale.com + ansible.builtin.copy: + dest: /etc/default/tailscaled + content: | + # Managed by ansible + PORT={{ network.tailscale.port }} + FLAGS="" + TS_NO_LOGS_NO_SUPPORT={{ not network.tailscale.telemetry }} + notify: Restart tailscaled + - name: Start Tailscale ansible.builtin.systemd: enabled: yes diff --git a/k8s/helm/headscale/values.yaml b/k8s/helm/headscale/values.yaml index f7725b42..530a9126 100644 --- a/k8s/helm/headscale/values.yaml +++ b/k8s/helm/headscale/values.yaml @@ -1,5 +1,18 @@ # Default values for headscale. +# Note: when deploying this under kubernetes with the haproxy-keepalived +# service (provided in this repo for bare-metal deployments), please add the +# following directive to the haproxy definition for port 443 ingress: +# +# timeout tunnel 30m +# +# Otherwise, you might see the following errors repeated about every 40 +# seconds in syslog from each tailscaled client: +# +# tailscaled[5712]: Received error: PollNetMap: unexpected EOF +# tailscaled[5712]: control: controlhttp: forcing port 443 dial due to recent noise dial +# tailscaled[5712]: [RATELIMIT] format("control: [v\x00JSON]%d%s") (2 dropped) + tlsHostname: headscale.example.com deployment: command: [ headscale, serve ] @@ -48,18 +61,11 @@ ingress: cert-manager.io/cluster-issuer: letsencrypt-prod kubernetes.io/ingress.class: nginx nginx.ingress.kubernetes.io/enable-access-log: "false" - nginx.ingress.kubernetes.io/keepalive-timeout: "300" - nginx.ingress.kubernetes.io/proxy-read-timeout: "86400s" - nginx.ingress.kubernetes.io/proxy-send-timeout: "86400s" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" nginx.ingress.kubernetes.io/configuration-snippet: | - proxy_set_header: Upgrade $http_upgrade; - proxy_set_header: Connection $connection_upgrade; - proxy_set_header: Host $server_name; - proxy_set_header: X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - nginx.ingress.kubernetes.io/proxy-buffering: "off" - nginx.ingress.kubernetes.io/proxy-http-version: "1.1" + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; rules: - host: headscale.example.com http: