From 6f99edd0a755ca11a4db040811002dad03ad2859 Mon Sep 17 00:00:00 2001
From: Wolfgang <mail@weirdrescue.pw>
Date: Wed, 22 Sep 2021 09:39:57 +0200
Subject: [PATCH] Replace Netatalk-based Time Machine with SMB, Replace HFS+
 with EXT4

---
 .gitignore                                    |  3 +-
 ansible.cfg                                   |  1 +
 group_vars/all/vars.yml                       | 47 +++++++++-----
 roles/avahi/tasks/main.yml                    |  8 ---
 roles/avahi/templates/netatalk.service        | 21 ------
 roles/essential/defaults/main.yml             |  3 +-
 roles/essential/tasks/main.yml                |  2 +
 roles/filesystems/mergerfs/tasks/main.yml     | 14 ++++
 .../mergerfs/templates/mergerfs-uncache.j2    | 23 +++++++
 roles/filesystems/mounts/tasks/main.yml       | 64 ++++++++-----------
 roles/filesystems/timemachine/tasks/main.yml  | 27 --------
 .../timemachine/templates/afp.conf            | 19 ------
 run.yml                                       |  8 +--
 tasks/user.yml                                |  2 +
 templates/global-include.conf                 |  9 ++-
 15 files changed, 112 insertions(+), 139 deletions(-)
 delete mode 100644 roles/avahi/templates/netatalk.service
 create mode 100644 roles/filesystems/mergerfs/templates/mergerfs-uncache.j2
 delete mode 100644 roles/filesystems/timemachine/tasks/main.yml
 delete mode 100644 roles/filesystems/timemachine/templates/afp.conf

diff --git a/.gitignore b/.gitignore
index eeecd721..aaa7b223 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,4 +4,5 @@
 .DS_Store
 secret.yml
 /group_vars/
-!/group_vars/all/vars.yml
\ No newline at end of file
+!/group_vars/all/vars.yml
+mountsraspi
\ No newline at end of file
diff --git a/ansible.cfg b/ansible.cfg
index be0487b9..266ebb3d 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -2,6 +2,7 @@
 INVENTORY = hosts
 vault_password_file = pass.sh
 ansible_python_interpreter=/usr/bin/python3
+timeout=30
 
 [ssh_connection]
 pipelining = True
diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml
index 1d201731..b9e7a543 100644
--- a/group_vars/all/vars.yml
+++ b/group_vars/all/vars.yml
@@ -17,6 +17,8 @@ ntp_timezone: "{{ timezone }}"
 
 locale: en_US.UTF-8
 
+fish_prompt_color: blue 
+
 keyboard_layout: us
 
 username: notthebee
@@ -71,7 +73,6 @@ msmtp_default_account: "mailbox"
 
 msmtp_alias_default : "{{ email }}"
 
-
 #
 # SSH (geerlingguy.security)
 #
@@ -85,8 +86,6 @@ security_autoupdate_mail_to: "{{ email }}"
 
 security_autoupdate_mail_on_error: true
 
-
-
 #
 # Docker
 #
@@ -116,6 +115,14 @@ enable_wireguard: true
 
 enable_ikev2: true
 
+#
+# Time Machine
+#
+timemachine_root: /mnt/timemachine
+
+timemachine_drive: /dev/disk/by-label/TimeMachine
+
+enable_timemachine: true
 
 #
 # IKEv2
@@ -136,8 +143,9 @@ samba_users:
 
 samba_global_include: global-include.conf
 
-samba_mitigate_cve_2017_7494: false
+samba_apple_extensions: yes 
 
+samba_mitigate_cve_2017_7494: false
 samba_shares:
   - name: Trash
     guest_ok: no
@@ -169,16 +177,21 @@ samba_shares:
     valid_users: "{{ username }}, {{ additional_users[0].name }}"
     write_list: "{{ username }}, {{ additional_users[0].name }}"
 
-
-#
-# Time Machine
-#
-timemachine_root: /mnt/timemachine
-
-timemachine_drive: /dev/disk/by-label/TimeMachine
-
-enable_timemachine: true
-
+  - name: TimeMachine
+    guest_ok: no
+    public: no
+    read_only: no
+    vfs_objects:
+      - name: fruit
+        options:
+        - name: time machine
+          value: 'yes'
+      - name: streams_xattr
+    writable: yes
+    browseable: yes
+    path: "{{ timemachine_root }}"
+    valid_users: "{{ username }}"
+    write_list: "{{ username }}"
 
 #
 # Snapraid and MergerFS
@@ -189,6 +202,9 @@ disks:
   - { path: /mnt/data1, src: /dev/disk/by-label/Data1, content: true }
   - { path: /mnt/data2, src: /dev/disk/by-label/Data2, content: true }
 
+cache_disks:
+  - { path: /mnt/cache1, src: /dev/disk/by-label/Cache1, content: true }
+
 parity_disks:
   - { path: /mnt/parity1, src: /dev/disk/by-label/Parity1, content: true }
 
@@ -211,7 +227,7 @@ snapraid_runner_smtp_port: "{{ email_smtp_port }}"
 snapraid_runner_command: "python3 {{ snapraid_runner_bin }} -c {{ snapraid_runner_conf }}"
 
 snapraid_runner_cron_jobs:
-  - { job: 'docker stop $(docker ps -a) && {{ snapraid_runner_command }} && docker start $(docker ps -a -q)', name: 'snapraid_runner', weekday: '*', hour: '04' }
+  - { job: 'docker stop $(docker ps -a) && {{ snapraid_runner_command }} && docker start $(docker ps -a -q)', name: 'snapraid_runner', weekday: '0', hour: '3' }
 
 snapraid_runner_delete_threshold: 5000
 
@@ -229,7 +245,6 @@ snapraid_config_excludes:
   - /tmp/
   - "/Downloads/*"
 
-
 #
 # SMART error reporting
 #
diff --git a/roles/avahi/tasks/main.yml b/roles/avahi/tasks/main.yml
index c74e55b8..fdc2acd2 100644
--- a/roles/avahi/tasks/main.yml
+++ b/roles/avahi/tasks/main.yml
@@ -20,14 +20,6 @@
     owner: root
     group: root
     mode: 0775
-
-- name: Copy the Netatalk config
-  template:
-    src: netatalk.service
-    dest: /etc/avahi/services/netatalk.service
-    owner: root
-    group: root
-    mode: 0775
   
 - name: Make sure the Avahi service is running and enabled
   service: 
diff --git a/roles/avahi/templates/netatalk.service b/roles/avahi/templates/netatalk.service
deleted file mode 100644
index 8e7f6bc9..00000000
--- a/roles/avahi/templates/netatalk.service
+++ /dev/null
@@ -1,21 +0,0 @@
-<?xml version="1.0" standalone='no'?><!--*-nxml-*-->
-<!DOCTYPE service-group SYSTEM "avahi-service.dtd">
-
-<service-group>
- <name replace-wildcards="yes">%h (AFP)</name>
-  <service>
-   <type>_afpovertcp._tcp</type>
-   <port>548</port>
-  </service>
-  <service>
-   <type>_device-info._tcp</type>
-   <port>0</port>
-   <txt-record>model=TimeCapsule</txt-record>
-  </service>
-  <service>
-   <type>_adisk._tcp</type>
-   <port>9</port>
-   <txt-record>sys=waMa=0,adVF=0x100,adVU=998BA54A-925C-4AB1-AEFF-1A274C70AFCE</txt-record>
-   <txt-record>dk0=adVN=TimeMachine,adVF=0x81</txt-record>
-  </service>
-</service-group>
\ No newline at end of file
diff --git a/roles/essential/defaults/main.yml b/roles/essential/defaults/main.yml
index 0b21d381..9643a7d4 100644
--- a/roles/essential/defaults/main.yml
+++ b/roles/essential/defaults/main.yml
@@ -10,4 +10,5 @@ extra_packages:
   - mosh
   - lm-sensors
   - iotop
-  - hddtemp
\ No newline at end of file
+  - hddtemp
+  - ncdu
\ No newline at end of file
diff --git a/roles/essential/tasks/main.yml b/roles/essential/tasks/main.yml
index 0f1d8677..d435d3c0 100644
--- a/roles/essential/tasks/main.yml
+++ b/roles/essential/tasks/main.yml
@@ -20,6 +20,7 @@
   with_items: 
     - iscsid
     - open-iscsi
+  ignore_errors: yes
 
 - name: Disable cron e-mail notifications
   cron:
@@ -34,6 +35,7 @@
     owner: root
     group: root
     mode: 0644
+  tags: mirrors
 
 - name: Update and upgrade apt packages
   become: true
diff --git a/roles/filesystems/mergerfs/tasks/main.yml b/roles/filesystems/mergerfs/tasks/main.yml
index 22ce49d1..1f91aabc 100644
--- a/roles/filesystems/mergerfs/tasks/main.yml
+++ b/roles/filesystems/mergerfs/tasks/main.yml
@@ -4,3 +4,17 @@
     name: "mergerfs"
     state: latest 
     update_cache: yes
+
+- name: Install the uncaching script
+  template:
+      src: mergerfs-uncache.j2
+      dest: /usr/local/bin/mergerfs-uncache
+      mode: 0775
+
+- name: Move data from cache drive to slow pool as it gets full
+  cron:
+    name: "uncache the mergerfs pool"
+    minute: "0"
+    hour: "3"
+    weekday: "1-6"
+    job: "/usr/local/bin/mergerfs-uncache"
\ No newline at end of file
diff --git a/roles/filesystems/mergerfs/templates/mergerfs-uncache.j2 b/roles/filesystems/mergerfs/templates/mergerfs-uncache.j2
new file mode 100644
index 00000000..16a67079
--- /dev/null
+++ b/roles/filesystems/mergerfs/templates/mergerfs-uncache.j2
@@ -0,0 +1,23 @@
+#!/bin/bash
+# Moves files away from the cache drive after they haven't been accessed for 5 days
+# Or when the cache drive is 85% full
+
+CACHE={{ cache_disks[0].path }}
+BACKING={{ mergerfs_root }}_slow
+PERCENTAGE=85
+N=5
+
+set -o errexit
+find "${CACHE}" -type f -atime +${N} -printf '%P\n' | \
+  rsync --files-from=- -axqHAXWES --preallocate --remove-source-files "${CACHE}/" "${BACKING}/"
+
+while [ $(df --output=pcent "${CACHE}" | grep -v Use | cut -d'%' -f1) -gt ${PERCENTAGE} ]
+do
+    FILE=$(find "${CACHE}" -type f -printf '%A@ %P\n' | \
+                  sort | \
+                  head -n 1 | \
+                  cut -d' ' -f2-)
+    test -n "${FILE}"
+    rsync -axqHAXWESR --preallocate --remove-source-files "${CACHE}/./${FILE}" "${BACKING}/"
+done
+
diff --git a/roles/filesystems/mounts/tasks/main.yml b/roles/filesystems/mounts/tasks/main.yml
index 33bfa82d..aea87020 100644
--- a/roles/filesystems/mounts/tasks/main.yml
+++ b/roles/filesystems/mounts/tasks/main.yml
@@ -8,6 +8,17 @@
   with_items: "{{ disks }}"
   when: disks[0] is defined
 
+- name: Mount the cache drives
+  mount:
+    name: "{{ item.path }}"
+    src: "{{ item.src }}"
+    fstype: ext4
+    state: mounted
+    opts: defaults,discard
+    passno: "2"
+  with_items: "{{ cache_disks }}"
+  when: cache_disks[0] is defined
+
 - name: Mount the parity drives
   mount:
     name: "{{ item.path }}"
@@ -18,11 +29,20 @@
   with_items: "{{ parity_disks }}"
   when: parity_disks[0] is defined
 
-- name: Mount the mergerfs array
+- name: Mount the mergerfs cached array
   mount:
     name: "{{ mergerfs_root }}"
+    src: /mnt/cache*:/mnt/data*
+    opts: category.create=lfs,direct_io,defaults,allow_other,moveonenospc=1,minfreespace=50G,fsname=mergerfs
+    fstype: fuse.mergerfs
+    state: mounted
+  when: disks[0] is defined
+
+- name: Mount the mergerfs array
+  mount:
+    name: "{{ mergerfs_root }}_slow"
     src: /mnt/data*
-    opts: direct_io,defaults,allow_other,minfreespace=50G,fsname=mergerfs
+    opts: direct_io,defaults,allow_other,moveonenospc=1,minfreespace=500G,fsname=mergerfs_slow
     fstype: fuse.mergerfs
     state: mounted
   when: disks[0] is defined
@@ -31,42 +51,8 @@
   mount:
     name: "{{ timemachine_root }}"
     src: "{{ timemachine_drive }}"
-    fstype: hfsplus
-    opts: defaults,nofail,force,rw
+    fstype: ext4
+    opts: defaults,discard
     state: mounted
     passno: "2"
-  when: enable_timemachine | default(False)
-
-- name: Remount the drive if it's read-only
-  when: enable_timemachine | default(False)
-  block:
-    - name: Check if the drive is read-only
-      shell:
-        cmd: grep "[[:space:]]ro[[:space:],]" /proc/mounts | grep timemachine
-      register: timemachine_ro
-      failed_when: timemachine_ro.rc == 0
-      changed_when: timemachine_ro.rc == 0
-
-  rescue:
-  - name: Stop the Netatalk service
-    service: 
-      name: netatalk
-      state: stopped
-    
-  - name: Unmount and fsck the drive if it became read-only
-    shell:
-      cmd: umount "{{ timemachine_drive }}" && fsck.hfsplus "{{ timemachine_drive }}"
-
-  - name: Mount the Time Machine drive again
-    mount:
-      name: "{{ timemachine_root }}"
-      src: "{{ timemachine_drive }}"
-      fstype: hfsplus
-      opts: defaults,nofail,force,rw
-      state: mounted
-      passno: "2"
-
-  - name: Start the Netatalk service
-    service: 
-      name: netatalk
-      state: started
\ No newline at end of file
+  when: enable_timemachine | default(False)
\ No newline at end of file
diff --git a/roles/filesystems/timemachine/tasks/main.yml b/roles/filesystems/timemachine/tasks/main.yml
deleted file mode 100644
index 30be3934..00000000
--- a/roles/filesystems/timemachine/tasks/main.yml
+++ /dev/null
@@ -1,27 +0,0 @@
----
-- name: Install required system packages
-  apt: 
-    name:
-      - netatalk
-      - hfsplus
-      - hfsprogs
-    state: latest 
-    update_cache: yes
-
-- name: Copy the Netatalk config
-  template:
-    src: afp.conf
-    dest: /etc/netatalk
-  register: netatalk_config
-
-- name: Make sure the Netatalk service is running and enabled
-  service: 
-    name: netatalk 
-    state: started 
-    enabled: yes
-
-- name: Restart Netatalk
-  service:
-    name: netatalk
-    state: restarted
-  when: netatalk_config.changed
\ No newline at end of file
diff --git a/roles/filesystems/timemachine/templates/afp.conf b/roles/filesystems/timemachine/templates/afp.conf
deleted file mode 100644
index 07af9c07..00000000
--- a/roles/filesystems/timemachine/templates/afp.conf
+++ /dev/null
@@ -1,19 +0,0 @@
-[Global]
-; Global server settings
-dbus daemon = /usr/bin/dbus-daemon
-disconnect time = 3
-sleep time = 2
-log file = /var/log/netatalk.log
-log level = default:info
-uam list = uams_dhx2_passwd.so
-zeroconf = no
-save password = no
-
-
-[TimeMachine]
-path = {{ timemachine_root }}
-time machine = yes
-spotlight = yes
-valid users = {{ username }}
-unix priv = yes
-perm = 0770
\ No newline at end of file
diff --git a/run.yml b/run.yml
index 24db4477..77886891 100644
--- a/run.yml
+++ b/run.yml
@@ -8,6 +8,8 @@
         - port
 
     - import_tasks: tasks/user.yml
+      tags:
+        - user
   
 - hosts: all
   become: yes
@@ -64,11 +66,6 @@
       tags: 
         - avahi
 
-    - role: filesystems/timemachine
-      tags:
-        - timemachine
-      when: enable_timemachine | default(False)
-
     - role: containers/watchtower
       tags:
         - watchtower
@@ -160,7 +157,6 @@
         - containers
       when: enable_smarthome | default(False)
 
-
     # SSH security (at the end because it breaks SSH connection)
     - role: geerlingguy.security
       tags: 
diff --git a/tasks/user.yml b/tasks/user.yml
index d7476ecd..135c0309 100644
--- a/tasks/user.yml
+++ b/tasks/user.yml
@@ -6,6 +6,8 @@
     password: "{{ password | password_hash('sha512') }}"
     groups: 
       - sudo
+      - users
+      - sambashare
     state: present
     append: true
     update_password: on_create
diff --git a/templates/global-include.conf b/templates/global-include.conf
index 64ceb5c2..40042b81 100644
--- a/templates/global-include.conf
+++ b/templates/global-include.conf
@@ -1 +1,8 @@
-multicast dns register = No
\ No newline at end of file
+multicast dns register = No
+log level = 1
+socket options = TCP_NODELAY SO_RCVBUF=65536 SO_SNDBUF=65536
+read raw = yes
+write raw = yes
+max xmit = 65535
+dead time = 15
+getwd cache = yes
\ No newline at end of file