From 6f99edd0a755ca11a4db040811002dad03ad2859 Mon Sep 17 00:00:00 2001 From: Wolfgang <mail@weirdrescue.pw> Date: Wed, 22 Sep 2021 09:39:57 +0200 Subject: [PATCH] Replace Netatalk-based Time Machine with SMB, Replace HFS+ with EXT4 --- .gitignore | 3 +- ansible.cfg | 1 + group_vars/all/vars.yml | 47 +++++++++----- roles/avahi/tasks/main.yml | 8 --- roles/avahi/templates/netatalk.service | 21 ------ roles/essential/defaults/main.yml | 3 +- roles/essential/tasks/main.yml | 2 + roles/filesystems/mergerfs/tasks/main.yml | 14 ++++ .../mergerfs/templates/mergerfs-uncache.j2 | 23 +++++++ roles/filesystems/mounts/tasks/main.yml | 64 ++++++++----------- roles/filesystems/timemachine/tasks/main.yml | 27 -------- .../timemachine/templates/afp.conf | 19 ------ run.yml | 8 +-- tasks/user.yml | 2 + templates/global-include.conf | 9 ++- 15 files changed, 112 insertions(+), 139 deletions(-) delete mode 100644 roles/avahi/templates/netatalk.service create mode 100644 roles/filesystems/mergerfs/templates/mergerfs-uncache.j2 delete mode 100644 roles/filesystems/timemachine/tasks/main.yml delete mode 100644 roles/filesystems/timemachine/templates/afp.conf diff --git a/.gitignore b/.gitignore index eeecd721..aaa7b223 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ .DS_Store secret.yml /group_vars/ -!/group_vars/all/vars.yml \ No newline at end of file +!/group_vars/all/vars.yml +mountsraspi \ No newline at end of file diff --git a/ansible.cfg b/ansible.cfg index be0487b9..266ebb3d 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -2,6 +2,7 @@ INVENTORY = hosts vault_password_file = pass.sh ansible_python_interpreter=/usr/bin/python3 +timeout=30 [ssh_connection] pipelining = True diff --git a/group_vars/all/vars.yml b/group_vars/all/vars.yml index 1d201731..b9e7a543 100644 --- a/group_vars/all/vars.yml +++ b/group_vars/all/vars.yml @@ -17,6 +17,8 @@ ntp_timezone: "{{ timezone }}" locale: en_US.UTF-8 +fish_prompt_color: blue + keyboard_layout: us username: notthebee @@ -71,7 +73,6 @@ msmtp_default_account: "mailbox" msmtp_alias_default : "{{ email }}" - # # SSH (geerlingguy.security) # @@ -85,8 +86,6 @@ security_autoupdate_mail_to: "{{ email }}" security_autoupdate_mail_on_error: true - - # # Docker # @@ -116,6 +115,14 @@ enable_wireguard: true enable_ikev2: true +# +# Time Machine +# +timemachine_root: /mnt/timemachine + +timemachine_drive: /dev/disk/by-label/TimeMachine + +enable_timemachine: true # # IKEv2 @@ -136,8 +143,9 @@ samba_users: samba_global_include: global-include.conf -samba_mitigate_cve_2017_7494: false +samba_apple_extensions: yes +samba_mitigate_cve_2017_7494: false samba_shares: - name: Trash guest_ok: no @@ -169,16 +177,21 @@ samba_shares: valid_users: "{{ username }}, {{ additional_users[0].name }}" write_list: "{{ username }}, {{ additional_users[0].name }}" - -# -# Time Machine -# -timemachine_root: /mnt/timemachine - -timemachine_drive: /dev/disk/by-label/TimeMachine - -enable_timemachine: true - + - name: TimeMachine + guest_ok: no + public: no + read_only: no + vfs_objects: + - name: fruit + options: + - name: time machine + value: 'yes' + - name: streams_xattr + writable: yes + browseable: yes + path: "{{ timemachine_root }}" + valid_users: "{{ username }}" + write_list: "{{ username }}" # # Snapraid and MergerFS @@ -189,6 +202,9 @@ disks: - { path: /mnt/data1, src: /dev/disk/by-label/Data1, content: true } - { path: /mnt/data2, src: /dev/disk/by-label/Data2, content: true } +cache_disks: + - { path: /mnt/cache1, src: /dev/disk/by-label/Cache1, content: true } + parity_disks: - { path: /mnt/parity1, src: /dev/disk/by-label/Parity1, content: true } @@ -211,7 +227,7 @@ snapraid_runner_smtp_port: "{{ email_smtp_port }}" snapraid_runner_command: "python3 {{ snapraid_runner_bin }} -c {{ snapraid_runner_conf }}" snapraid_runner_cron_jobs: - - { job: 'docker stop $(docker ps -a) && {{ snapraid_runner_command }} && docker start $(docker ps -a -q)', name: 'snapraid_runner', weekday: '*', hour: '04' } + - { job: 'docker stop $(docker ps -a) && {{ snapraid_runner_command }} && docker start $(docker ps -a -q)', name: 'snapraid_runner', weekday: '0', hour: '3' } snapraid_runner_delete_threshold: 5000 @@ -229,7 +245,6 @@ snapraid_config_excludes: - /tmp/ - "/Downloads/*" - # # SMART error reporting # diff --git a/roles/avahi/tasks/main.yml b/roles/avahi/tasks/main.yml index c74e55b8..fdc2acd2 100644 --- a/roles/avahi/tasks/main.yml +++ b/roles/avahi/tasks/main.yml @@ -20,14 +20,6 @@ owner: root group: root mode: 0775 - -- name: Copy the Netatalk config - template: - src: netatalk.service - dest: /etc/avahi/services/netatalk.service - owner: root - group: root - mode: 0775 - name: Make sure the Avahi service is running and enabled service: diff --git a/roles/avahi/templates/netatalk.service b/roles/avahi/templates/netatalk.service deleted file mode 100644 index 8e7f6bc9..00000000 --- a/roles/avahi/templates/netatalk.service +++ /dev/null @@ -1,21 +0,0 @@ -<?xml version="1.0" standalone='no'?><!--*-nxml-*--> -<!DOCTYPE service-group SYSTEM "avahi-service.dtd"> - -<service-group> - <name replace-wildcards="yes">%h (AFP)</name> - <service> - <type>_afpovertcp._tcp</type> - <port>548</port> - </service> - <service> - <type>_device-info._tcp</type> - <port>0</port> - <txt-record>model=TimeCapsule</txt-record> - </service> - <service> - <type>_adisk._tcp</type> - <port>9</port> - <txt-record>sys=waMa=0,adVF=0x100,adVU=998BA54A-925C-4AB1-AEFF-1A274C70AFCE</txt-record> - <txt-record>dk0=adVN=TimeMachine,adVF=0x81</txt-record> - </service> -</service-group> \ No newline at end of file diff --git a/roles/essential/defaults/main.yml b/roles/essential/defaults/main.yml index 0b21d381..9643a7d4 100644 --- a/roles/essential/defaults/main.yml +++ b/roles/essential/defaults/main.yml @@ -10,4 +10,5 @@ extra_packages: - mosh - lm-sensors - iotop - - hddtemp \ No newline at end of file + - hddtemp + - ncdu \ No newline at end of file diff --git a/roles/essential/tasks/main.yml b/roles/essential/tasks/main.yml index 0f1d8677..d435d3c0 100644 --- a/roles/essential/tasks/main.yml +++ b/roles/essential/tasks/main.yml @@ -20,6 +20,7 @@ with_items: - iscsid - open-iscsi + ignore_errors: yes - name: Disable cron e-mail notifications cron: @@ -34,6 +35,7 @@ owner: root group: root mode: 0644 + tags: mirrors - name: Update and upgrade apt packages become: true diff --git a/roles/filesystems/mergerfs/tasks/main.yml b/roles/filesystems/mergerfs/tasks/main.yml index 22ce49d1..1f91aabc 100644 --- a/roles/filesystems/mergerfs/tasks/main.yml +++ b/roles/filesystems/mergerfs/tasks/main.yml @@ -4,3 +4,17 @@ name: "mergerfs" state: latest update_cache: yes + +- name: Install the uncaching script + template: + src: mergerfs-uncache.j2 + dest: /usr/local/bin/mergerfs-uncache + mode: 0775 + +- name: Move data from cache drive to slow pool as it gets full + cron: + name: "uncache the mergerfs pool" + minute: "0" + hour: "3" + weekday: "1-6" + job: "/usr/local/bin/mergerfs-uncache" \ No newline at end of file diff --git a/roles/filesystems/mergerfs/templates/mergerfs-uncache.j2 b/roles/filesystems/mergerfs/templates/mergerfs-uncache.j2 new file mode 100644 index 00000000..16a67079 --- /dev/null +++ b/roles/filesystems/mergerfs/templates/mergerfs-uncache.j2 @@ -0,0 +1,23 @@ +#!/bin/bash +# Moves files away from the cache drive after they haven't been accessed for 5 days +# Or when the cache drive is 85% full + +CACHE={{ cache_disks[0].path }} +BACKING={{ mergerfs_root }}_slow +PERCENTAGE=85 +N=5 + +set -o errexit +find "${CACHE}" -type f -atime +${N} -printf '%P\n' | \ + rsync --files-from=- -axqHAXWES --preallocate --remove-source-files "${CACHE}/" "${BACKING}/" + +while [ $(df --output=pcent "${CACHE}" | grep -v Use | cut -d'%' -f1) -gt ${PERCENTAGE} ] +do + FILE=$(find "${CACHE}" -type f -printf '%A@ %P\n' | \ + sort | \ + head -n 1 | \ + cut -d' ' -f2-) + test -n "${FILE}" + rsync -axqHAXWESR --preallocate --remove-source-files "${CACHE}/./${FILE}" "${BACKING}/" +done + diff --git a/roles/filesystems/mounts/tasks/main.yml b/roles/filesystems/mounts/tasks/main.yml index 33bfa82d..aea87020 100644 --- a/roles/filesystems/mounts/tasks/main.yml +++ b/roles/filesystems/mounts/tasks/main.yml @@ -8,6 +8,17 @@ with_items: "{{ disks }}" when: disks[0] is defined +- name: Mount the cache drives + mount: + name: "{{ item.path }}" + src: "{{ item.src }}" + fstype: ext4 + state: mounted + opts: defaults,discard + passno: "2" + with_items: "{{ cache_disks }}" + when: cache_disks[0] is defined + - name: Mount the parity drives mount: name: "{{ item.path }}" @@ -18,11 +29,20 @@ with_items: "{{ parity_disks }}" when: parity_disks[0] is defined -- name: Mount the mergerfs array +- name: Mount the mergerfs cached array mount: name: "{{ mergerfs_root }}" + src: /mnt/cache*:/mnt/data* + opts: category.create=lfs,direct_io,defaults,allow_other,moveonenospc=1,minfreespace=50G,fsname=mergerfs + fstype: fuse.mergerfs + state: mounted + when: disks[0] is defined + +- name: Mount the mergerfs array + mount: + name: "{{ mergerfs_root }}_slow" src: /mnt/data* - opts: direct_io,defaults,allow_other,minfreespace=50G,fsname=mergerfs + opts: direct_io,defaults,allow_other,moveonenospc=1,minfreespace=500G,fsname=mergerfs_slow fstype: fuse.mergerfs state: mounted when: disks[0] is defined @@ -31,42 +51,8 @@ mount: name: "{{ timemachine_root }}" src: "{{ timemachine_drive }}" - fstype: hfsplus - opts: defaults,nofail,force,rw + fstype: ext4 + opts: defaults,discard state: mounted passno: "2" - when: enable_timemachine | default(False) - -- name: Remount the drive if it's read-only - when: enable_timemachine | default(False) - block: - - name: Check if the drive is read-only - shell: - cmd: grep "[[:space:]]ro[[:space:],]" /proc/mounts | grep timemachine - register: timemachine_ro - failed_when: timemachine_ro.rc == 0 - changed_when: timemachine_ro.rc == 0 - - rescue: - - name: Stop the Netatalk service - service: - name: netatalk - state: stopped - - - name: Unmount and fsck the drive if it became read-only - shell: - cmd: umount "{{ timemachine_drive }}" && fsck.hfsplus "{{ timemachine_drive }}" - - - name: Mount the Time Machine drive again - mount: - name: "{{ timemachine_root }}" - src: "{{ timemachine_drive }}" - fstype: hfsplus - opts: defaults,nofail,force,rw - state: mounted - passno: "2" - - - name: Start the Netatalk service - service: - name: netatalk - state: started \ No newline at end of file + when: enable_timemachine | default(False) \ No newline at end of file diff --git a/roles/filesystems/timemachine/tasks/main.yml b/roles/filesystems/timemachine/tasks/main.yml deleted file mode 100644 index 30be3934..00000000 --- a/roles/filesystems/timemachine/tasks/main.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -- name: Install required system packages - apt: - name: - - netatalk - - hfsplus - - hfsprogs - state: latest - update_cache: yes - -- name: Copy the Netatalk config - template: - src: afp.conf - dest: /etc/netatalk - register: netatalk_config - -- name: Make sure the Netatalk service is running and enabled - service: - name: netatalk - state: started - enabled: yes - -- name: Restart Netatalk - service: - name: netatalk - state: restarted - when: netatalk_config.changed \ No newline at end of file diff --git a/roles/filesystems/timemachine/templates/afp.conf b/roles/filesystems/timemachine/templates/afp.conf deleted file mode 100644 index 07af9c07..00000000 --- a/roles/filesystems/timemachine/templates/afp.conf +++ /dev/null @@ -1,19 +0,0 @@ -[Global] -; Global server settings -dbus daemon = /usr/bin/dbus-daemon -disconnect time = 3 -sleep time = 2 -log file = /var/log/netatalk.log -log level = default:info -uam list = uams_dhx2_passwd.so -zeroconf = no -save password = no - - -[TimeMachine] -path = {{ timemachine_root }} -time machine = yes -spotlight = yes -valid users = {{ username }} -unix priv = yes -perm = 0770 \ No newline at end of file diff --git a/run.yml b/run.yml index 24db4477..77886891 100644 --- a/run.yml +++ b/run.yml @@ -8,6 +8,8 @@ - port - import_tasks: tasks/user.yml + tags: + - user - hosts: all become: yes @@ -64,11 +66,6 @@ tags: - avahi - - role: filesystems/timemachine - tags: - - timemachine - when: enable_timemachine | default(False) - - role: containers/watchtower tags: - watchtower @@ -160,7 +157,6 @@ - containers when: enable_smarthome | default(False) - # SSH security (at the end because it breaks SSH connection) - role: geerlingguy.security tags: diff --git a/tasks/user.yml b/tasks/user.yml index d7476ecd..135c0309 100644 --- a/tasks/user.yml +++ b/tasks/user.yml @@ -6,6 +6,8 @@ password: "{{ password | password_hash('sha512') }}" groups: - sudo + - users + - sambashare state: present append: true update_password: on_create diff --git a/templates/global-include.conf b/templates/global-include.conf index 64ceb5c2..40042b81 100644 --- a/templates/global-include.conf +++ b/templates/global-include.conf @@ -1 +1,8 @@ -multicast dns register = No \ No newline at end of file +multicast dns register = No +log level = 1 +socket options = TCP_NODELAY SO_RCVBUF=65536 SO_SNDBUF=65536 +read raw = yes +write raw = yes +max xmit = 65535 +dead time = 15 +getwd cache = yes \ No newline at end of file