SYS-685 finalize network and monitoring settings for tailscale

README.md

@@ -66,9 +66,10 @@ The cluster-deployment tools here include helm charts and ansible playbooks to s
 | ddclient | [![](https://img.shields.io/docker/v/instantlinux/ddclient?sort=date)](https://hub.docker.com/r/instantlinux/ddclient "Version badge") | Dynamic DNS client |
 | ez-ipupdate | [![](https://img.shields.io/docker/v/instantlinux/ez-ipupdate?sort=date)](https://hub.docker.com/r/instantlinux/ez-ipupdate "Version badge") | Dynamic DNS client |
 | fluent-bit | ** | central logging for Kubernetes |
-| haproxy-keepalived | [![](https://img.shields.io/docker/v/instantlinux/haproxy-keepalived?sort=date)](https://hub.docker.com/r/instantlinux/haproxy-keepalived "Version badge") | load balancer |
+| garage | ** | S3-compatible object storage |
 | grafana | ** | monitoring dashboard with prometheus-based alerting |
 | guacamole | ** | authenticated remote-desktop server |
+| haproxy-keepalived | [![](https://img.shields.io/docker/v/instantlinux/haproxy-keepalived?sort=date)](https://hub.docker.com/r/instantlinux/haproxy-keepalived "Version badge") | load balancer |
 | headscale | ** | tailscale-compatible VPN control-plane |
 | mysqldump | [![](https://img.shields.io/docker/v/instantlinux/mysqldump?sort=date)](https://hub.docker.com/r/instantlinux/mysqldump "Version badge") | per-database alternative to xtrabackup |
 | nagios | [![](https://img.shields.io/docker/v/instantlinux/nagios?sort=date)](https://hub.docker.com/r/instantlinux/nagios "Version badge") | Nagios Core v4 for monitoring |

ansible/roles/monitoring_agent/handlers/main.yml

@@ -20,3 +20,6 @@
   service:
     name: snmpd
     state: restarted
+
+- name: Reload apparmor
+  ansible.builtin.command: apparmor_parser -r /etc/apparmor.d/usr.sbin.rsyslogd

ansible/roles/monitoring_agent/tasks/main.yml

@@ -4,41 +4,41 @@
 - import_tasks: packages.yml
 
 - name: Local nagios plugins path
-  file:
+  ansible.builtin.file:
     path: "{{ nagios.local_plugins_path }}"
     recurse: True
     state: directory
 
 - name: Install local NRPE plugins
-  copy:
+  ansible.builtin.copy:
     dest: "{{ nagios.local_plugins_path }}"
     force: no
     mode: 0755
     src: plugins/
 
 - name: Add NRPE to sudoers
-  template:
+  ansible.builtin.template:
     dest: /etc/sudoers.d/nrpe
     mode: 0440
     src: nrpe_sudoer.j2
     validate: "/usr/sbin/visudo -cf %s"
   notify: Restart NRPE
 
 - name: Allow from nagios server hosts
-  lineinfile:
+  ansible.builtin.lineinfile:
     dest: "{{ nagios.nrpe_cfg }}"
     line: allowed_hosts={{ ','.join(nagios.allowed_hosts) }}
     regexp: "^allowed_hosts=.*"
   notify: Restart NRPE
 
 - name: Add local nrpe definitions
-  template:
+  ansible.builtin.template:
     dest: "{{ nagios.nrpe_path }}"
     src: nrpe_command.cfg.j2
   notify: Restart NRPE
 
 - name: Set loglevel to INFO in snmpd
-  lineinfile:
+  ansible.builtin.lineinfile:
     dest: /etc/default/snmpd
     line: >
       SNMPDOPTS='-LS6d -Lf /dev/null -u snmp -g snmp
@@ -47,37 +47,45 @@
   notify: Restart snmpd
 
 - name: Enable NRPE service
-  service:
+  ansible.builtin.service:
     name: "{{ nagios.nrpe_service }}"
     enabled: yes
     state: started
 
 - name: Syslog host setup
-  template:
+  ansible.builtin.template:
     dest: /etc/rsyslog.d/custom.conf
     src: rsyslog-custom.conf.j2
   notify: Restart rsyslogd
 
 - name: Move imuxsock load out of stock /etc/rsyslog.conf
-  lineinfile:
+  ansible.builtin.lineinfile:
     dest: /etc/rsyslog.conf
     line: "# module(load=\"imuxsock\")"
     regexp: "^module.load=\"imuxsock\".*"
   notify: Restart rsyslogd
 
 - name: Ensure logs are in UTC
-  template:
+  ansible.builtin.template:
     dest: /etc/systemd/system/rsyslog.service
     src: rsyslog.service.j2
   notify: Restart rsyslogd
 
+- name: Allow rsyslogd access to cmdline
+  ansible.builtin.copy:
+    content: |
+      # Managed by ansible
+      /proc/[0-9]*/cmdline r,
+    dest: /etc/apparmor.d/local/usr.sbin.rsyslogd
+  notify: Reload apparmor
+
 - name: Put a leash on journald (default log files are huge)
-  template:
+  ansible.builtin.template:
     dest: /etc/systemd/journald.conf
     src: journald.conf.j2
   notify: Restart journald
 
 - name: Define tolerated drive errors
-  template:
+  ansible.builtin.template:
     dest: "{{ nagios.local_plugins_path }}/tolerations.yaml"
     src: tolerations.j2

ansible/roles/network/defaults/main.yml

@@ -20,12 +20,15 @@ network_defaults:
   - to: default
     via: "{{ hostvars[inventory_hostname]['ansible_default_ipv4']['gateway'] }}"
   search: "{{ hostvars[inventory_hostname]['ansible_dns']['search'] }}"
-  tailscale: false
+  tailscale:
+    enabled: false
+    port: 41641
+    telemetry: false
   cluster_ip: null
   vrrp_interface: null
 
 network_override: {}
-network: "{{ network_defaults | combine(network_override) }}"
+network: "{{ network_defaults | combine(network_override, recursive=True) }}"
 
 network_sysctl:
   net.ipv4.ip_nonlocal_bind: 1

ansible/roles/network/handlers/main.yml

@@ -1,14 +1,20 @@
 ---
 - name: Restart keepalived
-  service:
+  ansible.builtin.service:
     enabled: yes
     name: keepalived
     state: restarted
 
 - name: Netplan apply
-  command: netplan apply
+  ansible.builtin.command: netplan apply
 
 - name: Restart sshd
-  service:
+  ansible.builtin.service:
     name: ssh
     state: restarted
+
+- name: Restart tailscaled
+  ansible.builtin.service:
+    enabled: yes
+    name: tailscaled
+    state: restarted

ansible/roles/network/tasks/main.yml

@@ -10,7 +10,7 @@
   when: ansible_facts['distribution_version'] >= '18.04'
 
 - import_tasks: tailscale.yml
-  when: network.tailscale
+  when: network.tailscale.enabled
 
 - import_tasks: kernel.yml
 - import_tasks: sshd.yml

ansible/roles/network/tasks/tailscale.yml

@@ -17,6 +17,16 @@
     update_cache: true
     state: present
 
+- name: Disable telemetry to log.tailscale.com
+  ansible.builtin.copy:
+    dest: /etc/default/tailscaled
+    content: |
+      # Managed by ansible
+      PORT={{ network.tailscale.port }}
+      FLAGS=""
+      TS_NO_LOGS_NO_SUPPORT={{ not network.tailscale.telemetry }}
+  notify: Restart tailscaled
+
 - name: Start Tailscale
   ansible.builtin.systemd:
     enabled: yes

k8s/helm/headscale/values.yaml

@@ -1,5 +1,18 @@
 # Default values for headscale.
 
+# Note: when deploying this under kubernetes with the haproxy-keepalived
+#  service (provided in this repo for bare-metal deployments), please add the
+#  following directive to the haproxy definition for port 443 ingress:
+#
+#     timeout tunnel 30m
+#
+# Otherwise, you might see the following errors repeated about every 40
+#  seconds in syslog from each tailscaled client:
+#
+#  tailscaled[5712]: Received error: PollNetMap: unexpected EOF
+#  tailscaled[5712]: control: controlhttp: forcing port 443 dial due to recent noise dial
+#  tailscaled[5712]: [RATELIMIT] format("control: [v\x00JSON]%d%s") (2 dropped)
+
 tlsHostname: headscale.example.com
 deployment:
   command: [ headscale, serve ]
@@ -48,18 +61,11 @@ ingress:
     cert-manager.io/cluster-issuer: letsencrypt-prod
     kubernetes.io/ingress.class: nginx
     nginx.ingress.kubernetes.io/enable-access-log: "false"
-    nginx.ingress.kubernetes.io/keepalive-timeout: "300"
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "86400s"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "86400s"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
     nginx.ingress.kubernetes.io/configuration-snippet: |
-      proxy_set_header: Upgrade $http_upgrade;
-      proxy_set_header: Connection $connection_upgrade;
-      proxy_set_header: Host $server_name;
-      proxy_set_header: X-Real-IP $remote_addr;
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header X-Forwarded-Proto $scheme;
-    nginx.ingress.kubernetes.io/proxy-buffering: "off"
-    nginx.ingress.kubernetes.io/proxy-http-version: "1.1"
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection $connection_upgrade;
   rules:
   - host: headscale.example.com
     http: