Replace Netatalk-based Time Machine with SMB, Replace HFS+ with EXT4

.gitignore

@@ -4,4 +4,5 @@
 .DS_Store
 secret.yml
 /group_vars/
-!/group_vars/all/vars.yml
+!/group_vars/all/vars.yml
+mountsraspi

ansible.cfg

@@ -2,6 +2,7 @@
 INVENTORY = hosts
 vault_password_file = pass.sh
 ansible_python_interpreter=/usr/bin/python3
+timeout=30
 
 [ssh_connection]
 pipelining = True

group_vars/all/vars.yml

@@ -17,6 +17,8 @@ ntp_timezone: "{{ timezone }}"
 
 locale: en_US.UTF-8
 
+fish_prompt_color: blue 
+
 keyboard_layout: us
 
 username: notthebee
@@ -71,7 +73,6 @@ msmtp_default_account: "mailbox"
 
 msmtp_alias_default : "{{ email }}"
 
-
 #
 # SSH (geerlingguy.security)
 #
@@ -85,8 +86,6 @@ security_autoupdate_mail_to: "{{ email }}"
 
 security_autoupdate_mail_on_error: true
 
-
-
 #
 # Docker
 #
@@ -116,6 +115,14 @@ enable_wireguard: true
 
 enable_ikev2: true
 
+#
+# Time Machine
+#
+timemachine_root: /mnt/timemachine
+
+timemachine_drive: /dev/disk/by-label/TimeMachine
+
+enable_timemachine: true
 
 #
 # IKEv2
@@ -136,8 +143,9 @@ samba_users:
 
 samba_global_include: global-include.conf
 
-samba_mitigate_cve_2017_7494: false
+samba_apple_extensions: yes 
 
+samba_mitigate_cve_2017_7494: false
 samba_shares:
   - name: Trash
     guest_ok: no
@@ -169,16 +177,21 @@ samba_shares:
     valid_users: "{{ username }}, {{ additional_users[0].name }}"
     write_list: "{{ username }}, {{ additional_users[0].name }}"
 
-
-#
-# Time Machine
-#
-timemachine_root: /mnt/timemachine
-
-timemachine_drive: /dev/disk/by-label/TimeMachine
-
-enable_timemachine: true
-
+  - name: TimeMachine
+    guest_ok: no
+    public: no
+    read_only: no
+    vfs_objects:
+      - name: fruit
+        options:
+        - name: time machine
+          value: 'yes'
+      - name: streams_xattr
+    writable: yes
+    browseable: yes
+    path: "{{ timemachine_root }}"
+    valid_users: "{{ username }}"
+    write_list: "{{ username }}"
 
 #
 # Snapraid and MergerFS
@@ -189,6 +202,9 @@ disks:
   - { path: /mnt/data1, src: /dev/disk/by-label/Data1, content: true }
   - { path: /mnt/data2, src: /dev/disk/by-label/Data2, content: true }
 
+cache_disks:
+  - { path: /mnt/cache1, src: /dev/disk/by-label/Cache1, content: true }
+
 parity_disks:
   - { path: /mnt/parity1, src: /dev/disk/by-label/Parity1, content: true }
 
@@ -211,7 +227,7 @@ snapraid_runner_smtp_port: "{{ email_smtp_port }}"
 snapraid_runner_command: "python3 {{ snapraid_runner_bin }} -c {{ snapraid_runner_conf }}"
 
 snapraid_runner_cron_jobs:
-  - { job: 'docker stop $(docker ps -a) && {{ snapraid_runner_command }} && docker start $(docker ps -a -q)', name: 'snapraid_runner', weekday: '*', hour: '04' }
+  - { job: 'docker stop $(docker ps -a) && {{ snapraid_runner_command }} && docker start $(docker ps -a -q)', name: 'snapraid_runner', weekday: '0', hour: '3' }
 
 snapraid_runner_delete_threshold: 5000
 
@@ -229,7 +245,6 @@ snapraid_config_excludes:
   - /tmp/
   - "/Downloads/*"
 
-
 #
 # SMART error reporting
 #

roles/avahi/tasks/main.yml

@@ -20,14 +20,6 @@
     owner: root
     group: root
     mode: 0775
-
-- name: Copy the Netatalk config
-  template:
-    src: netatalk.service
-    dest: /etc/avahi/services/netatalk.service
-    owner: root
-    group: root
-    mode: 0775
 
 - name: Make sure the Avahi service is running and enabled
   service: 

roles/essential/defaults/main.yml

@@ -10,4 +10,5 @@ extra_packages:
   - mosh
   - lm-sensors
   - iotop
-  - hddtemp
+  - hddtemp
+  - ncdu

roles/essential/tasks/main.yml

@@ -20,6 +20,7 @@
   with_items: 
     - iscsid
     - open-iscsi
+  ignore_errors: yes
 
 - name: Disable cron e-mail notifications
   cron:
@@ -34,6 +35,7 @@
     owner: root
     group: root
     mode: 0644
+  tags: mirrors
 
 - name: Update and upgrade apt packages
   become: true

roles/filesystems/mergerfs/tasks/main.yml

@@ -4,3 +4,17 @@
     name: "mergerfs"
     state: latest 
     update_cache: yes
+
+- name: Install the uncaching script
+  template:
+      src: mergerfs-uncache.j2
+      dest: /usr/local/bin/mergerfs-uncache
+      mode: 0775
+
+- name: Move data from cache drive to slow pool as it gets full
+  cron:
+    name: "uncache the mergerfs pool"
+    minute: "0"
+    hour: "3"
+    weekday: "1-6"
+    job: "/usr/local/bin/mergerfs-uncache"

roles/filesystems/mergerfs/templates/mergerfs-uncache.j2

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Moves files away from the cache drive after they haven't been accessed for 5 days
+# Or when the cache drive is 85% full
+
+CACHE={{ cache_disks[0].path }}
+BACKING={{ mergerfs_root }}_slow
+PERCENTAGE=85
+N=5
+
+set -o errexit
+find "${CACHE}" -type f -atime +${N} -printf '%P\n' | \
+  rsync --files-from=- -axqHAXWES --preallocate --remove-source-files "${CACHE}/" "${BACKING}/"
+
+while [ $(df --output=pcent "${CACHE}" | grep -v Use | cut -d'%' -f1) -gt ${PERCENTAGE} ]
+do
+    FILE=$(find "${CACHE}" -type f -printf '%A@ %P\n' | \
+                  sort | \
+                  head -n 1 | \
+                  cut -d' ' -f2-)
+    test -n "${FILE}"
+    rsync -axqHAXWESR --preallocate --remove-source-files "${CACHE}/./${FILE}" "${BACKING}/"
+done
+

roles/filesystems/mounts/tasks/main.yml

@@ -8,6 +8,17 @@
   with_items: "{{ disks }}"
   when: disks[0] is defined
 
+- name: Mount the cache drives
+  mount:
+    name: "{{ item.path }}"
+    src: "{{ item.src }}"
+    fstype: ext4
+    state: mounted
+    opts: defaults,discard
+    passno: "2"
+  with_items: "{{ cache_disks }}"
+  when: cache_disks[0] is defined
+
 - name: Mount the parity drives
   mount:
     name: "{{ item.path }}"
@@ -18,11 +29,20 @@
   with_items: "{{ parity_disks }}"
   when: parity_disks[0] is defined
 
-- name: Mount the mergerfs array
+- name: Mount the mergerfs cached array
   mount:
     name: "{{ mergerfs_root }}"
+    src: /mnt/cache*:/mnt/data*
+    opts: category.create=lfs,direct_io,defaults,allow_other,moveonenospc=1,minfreespace=50G,fsname=mergerfs
+    fstype: fuse.mergerfs
+    state: mounted
+  when: disks[0] is defined
+
+- name: Mount the mergerfs array
+  mount:
+    name: "{{ mergerfs_root }}_slow"
     src: /mnt/data*
-    opts: direct_io,defaults,allow_other,minfreespace=50G,fsname=mergerfs
+    opts: direct_io,defaults,allow_other,moveonenospc=1,minfreespace=500G,fsname=mergerfs_slow
     fstype: fuse.mergerfs
     state: mounted
   when: disks[0] is defined
@@ -31,42 +51,8 @@
   mount:
     name: "{{ timemachine_root }}"
     src: "{{ timemachine_drive }}"
-    fstype: hfsplus
-    opts: defaults,nofail,force,rw
+    fstype: ext4
+    opts: defaults,discard
     state: mounted
     passno: "2"
-  when: enable_timemachine | default(False)
-
-- name: Remount the drive if it's read-only
-  when: enable_timemachine | default(False)
-  block:
-    - name: Check if the drive is read-only
-      shell:
-        cmd: grep "[[:space:]]ro[[:space:],]" /proc/mounts | grep timemachine
-      register: timemachine_ro
-      failed_when: timemachine_ro.rc == 0
-      changed_when: timemachine_ro.rc == 0
-
-  rescue:
-  - name: Stop the Netatalk service
-    service: 
-      name: netatalk
-      state: stopped
-
-  - name: Unmount and fsck the drive if it became read-only
-    shell:
-      cmd: umount "{{ timemachine_drive }}" && fsck.hfsplus "{{ timemachine_drive }}"
-
-  - name: Mount the Time Machine drive again
-    mount:
-      name: "{{ timemachine_root }}"
-      src: "{{ timemachine_drive }}"
-      fstype: hfsplus
-      opts: defaults,nofail,force,rw
-      state: mounted
-      passno: "2"
-
-  - name: Start the Netatalk service
-    service: 
-      name: netatalk
-      state: started
+  when: enable_timemachine | default(False)