reload mirror jobs online

README.md

@@ -5,5 +5,5 @@ tunasync
 
 - [ ] status file
 - [x] btrfs backend (create snapshot before syncing)
-- [ ] add mirror job online
+- [x] add mirror job online
 - [ ] debmirror provider

examples/tunasync.ini

@@ -2,30 +2,31 @@
 log_dir = /var/log/tunasync
 ; mirror_root = /srv/mirror_disk
 mirror_root = /mnt/sdb1/mirror
-use_btrfs = yes
+use_btrfs = no
 local_dir = {mirror_root}/{mirror_name}/_working
 ; maximum numbers of running jobs
-concurrent = 3
+concurrent = 2
 ; interval in minutes
-interval = 120
+interval = 1
+max_retry = 2
 
 [btrfs]
 service_dir = {mirror_root}/{mirror_name}/_current
 working_dir = {mirror_root}/{mirror_name}/_working
 tmp_dir = {mirror_root}/{mirror_name}/_tmp
 
 
+# rmirror:archlinux]
+# provider = rsync
+# upstream = rsync://mirror.us.leaseweb.net/archlinux/
+# log_file = /tmp/archlinux-{date}.log
+# use_ipv6 = yes
+
 [mirror:archlinux]
-provider = rsync
-upstream = rsync://mirror.us.leaseweb.net/archlinux/
+provider = shell
+command = sleep 20
+local_dir = /mnt/sdb1/mirror/archlinux/current/
 log_file = /tmp/archlinux-{date}.log
-use_ipv6 = yes
-
-# [mirror:archlinux]
-# provider = shell
-# command = sleep 10
-# local_dir = /mnt/sdb1/mirror/archlinux/current/
-# log_file = /tmp/archlinux-{date}.log
 
 
 [mirror:arch2]
@@ -40,3 +41,9 @@ provider = shell
 command = ./shell_provider.sh
 log_file = /tmp/arch3-{date}.log
 use_btrfs = no
+
+[mirror:arch4]
+provider = shell
+command = ./shell_provider.sh
+log_file = /tmp/arch4-{date}.log
+use_btrfs = no

tunasync/btrfs_snapshot.py

@@ -2,13 +2,14 @@
 # -*- coding:utf-8 -*-
 import sh
 import os
+from .hook import JobHook
 
 
 class BtrfsVolumeError(Exception):
     pass
 
 
-class BtrfsHook(object):
+class BtrfsHook(JobHook):
 
     def __init__(self, service_dir, working_dir, tmp_dir):
         self.service_dir = service_dir

tunasync/hook.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python2
+# -*- coding:utf-8 -*-
+
+
+class JobHook(object):
+
+    def before_job(self):
+        raise NotImplementedError("")
+
+    def after_job(self):
+        raise NotImplementedError("")
+
+# vim: ts=4 sw=4 sts=4 expandtab

tunasync/jobs.py

@@ -1,22 +1,49 @@
 #!/usr/bin/env python2
 # -*- coding:utf-8 -*-
+import sys
 import time
+import signal
 
 
-def run_job(sema, provider):
+def run_job(sema, child_q, manager_q, provider):
+    aquired = False
+
+    def before_quit(*args):
+        provider.terminate()
+        if aquired:
+            print("{} release semaphore".format(provider.name))
+            sema.release()
+        sys.exit(0)
+
+    signal.signal(signal.SIGTERM, before_quit)
+
     while 1:
-        sema.acquire(True)
+        try:
+            sema.acquire(True)
+        except:
+            break
+        aquired = True
         print("start syncing {}".format(provider.name))
 
         for hook in provider.hooks:
             hook.before_job()
 
         provider.run()
+        provider.wait()
 
         for hook in provider.hooks[::-1]:
             hook.after_job()
 
         sema.release()
+        aquired = False
+        try:
+            msg = child_q.get(timeout=1)
+            if msg == "terminate":
+                manager_q.put((provider.name, "QUIT"))
+                break
+        except:
+            pass
+
         print("syncing {} finished, sleep {} minutes for the next turn".format(
             provider.name, provider.interval
         ))

tunasync/mirror_provider.py

@@ -17,10 +17,22 @@ def __init__(self, name, local_dir, log_file="/dev/null",
         self.log_file = log_file
         self.interval = interval
         self.hooks = hooks
+        self.p = None
 
     def run(self):
         raise NotImplementedError("run method should be implemented")
 
+    def terminate(self):
+        if self.p is not None:
+            self.p.process.terminate()
+            print("{} terminated".format(self.name))
+            self.p = None
+
+    def wait(self):
+        if self.p is not None:
+            self.p.wait()
+            self.p = None
+
 
 class RsyncProvider(MirrorProvider):
 
@@ -60,7 +72,8 @@ def run(self):
         now = datetime.now().strftime("%Y-%m-%d_%H")
         log_file = self.log_file.format(date=now)
 
-        sh.rsync(*_args, _out=log_file, _err=log_file, _out_bufsize=1)
+        self.p = sh.rsync(*_args, _out=log_file, _err=log_file,
+                          _out_bufsize=1, _bg=True)
 
 
 class ShellProvider(MirrorProvider):
@@ -78,14 +91,16 @@ def run(self):
         log_file = self.log_file.format(date=now)
 
         new_env = os.environ.copy()
+        new_env["TUNASYNC_MIRROR_NAME"] = self.name
         new_env["TUNASYNC_LOCAL_DIR"] = self.local_dir
         new_env["TUNASYNC_LOG_FILE"] = log_file
 
         _cmd = self.command[0]
         _args = [] if len(self.command) == 1 else self.command[1:]
 
         cmd = sh.Command(_cmd)
-        cmd(*_args, _env=new_env, _out=log_file, _err=log_file, _out_bufsize=1)
+        self.p = cmd(*_args, _env=new_env, _out=log_file,
+                     _err=log_file, _out_bufsize=1, _bg=True)
 
 
 # vim: ts=4 sw=4 sts=4 expandtab