Added steps

forkbuntu/__init__.py

@@ -1,11 +1,21 @@
-from . import controllers, services
+from . import controllers, services, steps
 from cfoundation import create_app
+from munch import munchify
 from os import path
 from pydash import _
 from tempfile import mkdtemp
 import os
+import re
 import yaml
-from munch import munchify
+
+def get_steps(app):
+    context = Object()
+    for key in dir(steps):
+        matches = re.findall(r'[A-Z].*', key)
+        if len(matches) > 0:
+            name = _.snake_case(key)
+            setattr(context, name, getattr(steps, key)(name, app))
+    return context
 
 def load_conf(conf):
     with open(path.abspath('config.yml'), 'r') as f:
@@ -38,6 +48,7 @@ def load_conf(conf):
         'paths': {
             'apt_ftparchive': '.tmp/apt-ftparchive',
             'cwd': os.getcwd(),
+            'cwt': '.tmp',
             'filesystem': '.tmp/filesystem',
             'indices': '.tmp/indices',
             'iso': 'ubuntu-18.04-server-amd64.iso',
@@ -49,3 +60,6 @@ def load_conf(conf):
         }
     })
 )
+
+class Object():
+    pass

forkbuntu/__main__.py

@@ -1,9 +1,11 @@
 import os, sys
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
 from forkbuntu import App
+from forkbuntu import get_steps
 
 def main():
     with App() as app:
+        app.steps = get_steps(app)
         app.run()
 
 if __name__ == '__main__':

forkbuntu/controllers/__init__.py

@@ -1 +1,2 @@
 from .base import Base
+from .dev import Dev

forkbuntu/controllers/base.py

@@ -3,7 +3,6 @@
 from halo import Halo
 from munch import munchify
 from pydash import _
-import logging
 import json
 
 class Base(Controller):
@@ -13,63 +12,6 @@ class Meta:
 
     @expose()
     def default(self):
-        log = self.app.log
-        c = self.app.conf
-        s = self.app.services
-        spinner = self.app.spinner
-        spinner.start('initializing')
-        s.setup.init()
-        spinner.succeed('initialized')
-        setattr(self.app, 'gpg_keys', s.gpg.get_keys())
-        gpg_keys = self.app.gpg_keys
-        log.debug('gpg_keys: ' + json.dumps(gpg_keys, indent=4, sort_keys=True))
-        spinner.start('unpacking iso')
-        s.unpack.iso()
-        spinner.succeed('unpacked iso')
-        spinner.start('unpacking filesystem')
-        s.unpack.filesystem()
-        spinner.succeed('unpacked filesystem')
-        spinner.start('loading config')
-        setattr(self.app, 'release', s.configure.load_release())
-        release = self.app.release
-        log.debug('release: ' + json.dumps(release, indent=4, sort_keys=True))
-        self.app.conf = munchify(_.merge({}, self.app.conf, {
-            'codename': c.codename if 'codename' in c else release.distrib_codename,
-            'distrib_id': c.distrib_id if 'distrib_id' in c else release.distrib_id,
-            'name': c.name if 'name' in c else release.distrib_name,
-            'version': c.version if 'version' in c else release.distrib_version
-        }))
-        self.app.conf = munchify(_.merge({}, self.app.conf, {
-            'description': c.description if 'description' in c else c.name + ' ' + c.version,
-            'hostname': c.hostname if 'hostname' in c else _.snake_case(c.name)
-        }))
-        log.debug('conf: ' + json.dumps(self.app.conf, indent=4, sort_keys=True))
-        spinner.succeed('loaded config')
-        spinner.start('merging iso')
-        s.configure.merge_iso()
-        spinner.succeed('merged iso')
-        spinner.start('merging filesystem')
-        s.configure.merge_filesystem()
-        spinner.succeed('merged filesystem')
-        spinner.start('building keyring')
-        s.gpg.build_keyring()
-        spinner.succeed('built keyring')
-        spinner.start('creating extras')
-        s.extras.create()
-        spinner.succeed('created extras')
-        spinner.start('configuring filesystem')
-        s.configure.filesystem()
-        spinner.succeed('configured filesystem')
-        spinner.start('signing iso')
-        s.configure.sign_iso()
-        spinner.succeed('signed iso')
-        spinner.start('packing filesystem')
-        s.pack.filesystem()
-        spinner.succeed('packed filesystem')
-        spinner.start('packing iso')
-        s.pack.iso()
-        spinner.succeed('packed iso')
-        spinner.start('cleaning')
-        s.clean.tmp()
-        spinner.succeed('cleaned')
+        steps = self.app.steps
+        steps.clean.start()
         spinner.start('iso created: ' + c.paths.output).succeed()

forkbuntu/controllers/dev.py

@@ -0,0 +1,16 @@
+from cement.core.controller import expose
+from cfoundation import Controller
+import json
+
+class Dev(Controller):
+    class Meta:
+        description = 'Development'
+        label = 'dev'
+        stacked_on = 'base'
+        stacked_type = 'nested'
+
+    @expose()
+    def default(self):
+        log = self.app.log
+        c = self.app.conf
+        s = self.app.services

forkbuntu/services/__init__.py

@@ -1,3 +1,4 @@
+from .cache import Cache
 from .clean import Clean
 from .configure import Configure
 from .extras import Extras

forkbuntu/services/cache.py

@@ -0,0 +1,44 @@
+from cfoundation import Service
+from munch import munchify, Munch, unmunchify
+from os import path
+import checksum
+import checksumdir
+import hashlib
+import yaml
+
+class Cache(Service):
+    def checksum(self, checksum_path):
+        value = None
+        if path.exists(checksum_path):
+            if path.isdir(checksum_path):
+                value = checksumdir.dirhash(checksum_path)
+            else:
+                value = checksum.get_for_file(checksum_path).decode('utf-8')
+        return value
+
+    def register(self, key):
+        c = self.app.conf
+        step = getattr(self.app.steps, key)
+        cache = self.get()
+        cache[key] = []
+        for checksum_path in step.checksum_paths:
+            cache[key].append(self.checksum(checksum_path))
+        with open(path.join(c.paths.cwt, '.cache.yml'), 'w') as f:
+            yaml.dump(unmunchify(cache), f, default_flow_style=False)
+        return cache
+
+    def get(self, key=None):
+        c = self.app.conf
+        cache_path = path.join(c.paths.cwt, '.cache.yml')
+        cache = Munch()
+        if not path.exists(cache_path):
+            return cache
+        with open(cache_path, 'r') as f:
+            data = munchify(yaml.load(f))
+            if data:
+                cache = data
+        if key:
+            if not key in cache:
+                return []
+            return cache[key]
+        return cache

forkbuntu/services/clean.py

@@ -5,3 +5,6 @@ class Clean(Service):
     def tmp(self):
         c = self.app.conf
         shutil.rmtree(c.paths.tmp)
+
+    def nuke(self):
+        shutil.rmtree(path.join(c.paths.cwd, '.tmp'))

forkbuntu/step.py

@@ -0,0 +1,58 @@
+from pydash import _
+
+steps_run = []
+
+class Step():
+    global steps_run
+
+    def __init__(self, name, app):
+        self.name = name
+        self.app = app
+        self.log = app.log
+
+    def run_required(self):
+        steps = self.app.steps
+        if not hasattr(self, 'requires'):
+            return None
+        for required in self.requires:
+            if not _.includes(steps_run, required):
+                getattr(steps, required).start()
+
+    def checksum(self):
+        s = self.app.services
+        if not hasattr(self, 'checksum_paths'):
+            return False
+        for checksum_path in self.checksum_paths:
+            checksum = s.cache.checksum(checksum_path)
+            if not _.includes(s.cache.get(self.name), checksum):
+                return False
+        return True
+
+    def register(self):
+        s = self.app.services
+        if hasattr(self, 'checksum_paths'):
+            s.cache.register(self.name)
+
+    def start(self):
+        spinner = self.app.spinner
+        self.run_required()
+        spinner.start(self.messages.present)
+        passed = self.checksum()
+        if passed:
+            self.cached()
+            return None
+        self.run()
+        self.register()
+        self.finish()
+
+    def cached(self):
+        global steps_run
+        spinner = self.app.spinner
+        steps_run.append(self.name)
+        spinner.warn(self.messages.cache)
+
+    def finish(self):
+        global steps_run
+        spinner = self.app.spinner
+        steps_run.append(self.name)
+        spinner.succeed(self.messages.past)

forkbuntu/steps/__init__.py

@@ -0,0 +1,13 @@
+from .build_keyring import BuildKeyring
+from .clean import Clean
+from .configure_filesystem import ConfigureFilesystem
+from .create_extras import CreateExtras
+from .initialize import Initialize
+from .load_config import LoadConfig
+from .merge_filesystem import MergeFilesystem
+from .merge_iso import MergeIso
+from .pack_filesystem import PackFilesystem
+from .pack_iso import PackIso
+from .sign_iso import SignIso
+from .unpack_filesystem import UnpackFilesystem
+from .unpack_iso import UnpackIso

forkbuntu/steps/build_keyring.py

@@ -0,0 +1,22 @@
+from ..step import Step
+from munch import munchify
+from os import path
+
+class BuildKeyring(Step):
+    messages = munchify({
+        'past': 'built keyring',
+        'present': 'building keyring',
+        'cache': 'using built keyring cache'
+    })
+    requires = [
+        'merge_filesystem'
+    ]
+
+    def __init__(self, name, app):
+        super().__init__(name, app)
+        c = app.conf
+        self.checksum_paths = [path.join(c.paths.iso)]
+
+    def run(self):
+        s = self.app.services
+        s.gpg.build_keyring()

forkbuntu/steps/clean.py

@@ -0,0 +1,16 @@
+from ..step import Step
+from munch import munchify
+from os import path
+
+class Clean(Step):
+    messages = munchify({
+        'past': 'cleaned',
+        'present': 'cleaning'
+    })
+    requires = [
+        'pack_iso'
+    ]
+
+    def run(self):
+        s = self.app.services
+        s.clean.tmp()

forkbuntu/steps/configure_filesystem.py

@@ -0,0 +1,22 @@
+from ..step import Step
+from munch import munchify
+from os import path
+
+class ConfigureFilesystem(Step):
+    messages = munchify({
+        'past': 'configured filesystem',
+        'present': 'configuring filesystem',
+        'cache': 'using configured filesystem cache'
+    })
+    requires = [
+        'create_extras'
+    ]
+
+    def __init__(self, name, app):
+        super().__init__(name, app)
+        c = app.conf
+        self.checksum_paths = [path.join(c.paths.iso)]
+
+    def run(self):
+        s = self.app.services
+        s.configure.filesystem()

forkbuntu/steps/create_extras.py

@@ -0,0 +1,22 @@
+from ..step import Step
+from munch import munchify
+from os import path
+
+class CreateExtras(Step):
+    messages = munchify({
+        'past': 'created extras',
+        'present': 'creating extras',
+        'cache': 'using extras cache'
+    })
+    requires = [
+        'build_keyring'
+    ]
+
+    def __init__(self, name, app):
+        super().__init__(name, app)
+        c = app.conf
+        self.checksum_paths = [path.join(c.paths.iso)]
+
+    def run(self):
+        s = self.app.services
+        s.extras.create()

forkbuntu/steps/initialize.py

@@ -0,0 +1,21 @@
+from ..step import Step
+from munch import munchify
+from time import sleep
+import json
+
+class Initialize(Step):
+    messages = munchify({
+        'past': 'initialized',
+        'present': 'initializing'
+    })
+
+    def run(self):
+        s = self.app.services
+        s.setup.init()
+        setattr(self.app, 'gpg_keys', s.gpg.get_keys())
+        gpg_keys = self.app.gpg_keys
+
+    def finish(self):
+        super().finish()
+        log = self.app.log
+        log.debug('gpg_keys: ' + json.dumps(self.app.gpg_keys, indent=4, sort_keys=True))