|
| 1 | +######################################################################## |
| 2 | +# |
| 3 | +# Cache-wrapper for a function or class. |
| 4 | +# |
| 5 | +# Save the result of calling a function or creating an object-instance |
| 6 | +# to harddisk. This is used to persist the data so it can be reloaded |
| 7 | +# very quickly and easily. |
| 8 | +# |
| 9 | +# Implemented in Python 3.5 |
| 10 | +# |
| 11 | +######################################################################## |
| 12 | +# |
| 13 | +# This file is part of the TensorFlow Tutorials available at: |
| 14 | +# |
| 15 | +# https://github.com/Hvass-Labs/TensorFlow-Tutorials |
| 16 | +# |
| 17 | +# Published under the MIT License. See the file LICENSE for details. |
| 18 | +# |
| 19 | +# Copyright 2016 by Magnus Erik Hvass Pedersen |
| 20 | +# |
| 21 | +######################################################################## |
| 22 | + |
| 23 | +import os |
| 24 | +import pickle |
| 25 | +import numpy as np |
| 26 | + |
| 27 | +######################################################################## |
| 28 | + |
| 29 | + |
| 30 | +def cache(cache_path, fn, *args, **kwargs): |
| 31 | + """ |
| 32 | + Cache-wrapper for a function or class. If the cache-file exists |
| 33 | + then the data is reloaded and returned, otherwise the function |
| 34 | + is called and the result is saved to cache. The fn-argument can |
| 35 | + also be a class instead, in which case an object-instance is |
| 36 | + created and saved to the cache-file. |
| 37 | +
|
| 38 | + :param cache_path: |
| 39 | + File-path for the cache-file. |
| 40 | +
|
| 41 | + :param fn: |
| 42 | + Function or class to be called. |
| 43 | +
|
| 44 | + :param args: |
| 45 | + Arguments to the function or class-init. |
| 46 | +
|
| 47 | + :param kwargs: |
| 48 | + Keyword arguments to the function or class-init. |
| 49 | +
|
| 50 | + :return: |
| 51 | + The result of calling the function or creating the object-instance. |
| 52 | + """ |
| 53 | + |
| 54 | + # If the cache-file exists. |
| 55 | + if os.path.exists(cache_path): |
| 56 | + # Load the cached data from the file. |
| 57 | + with open(cache_path, mode='rb') as file: |
| 58 | + obj = pickle.load(file) |
| 59 | + |
| 60 | + print("- Data loaded from cache-file: " + cache_path) |
| 61 | + else: |
| 62 | + # The cache-file does not exist. |
| 63 | + |
| 64 | + # Call the function / class-init with the supplied arguments. |
| 65 | + obj = fn(*args, **kwargs) |
| 66 | + |
| 67 | + # Save the data to a cache-file. |
| 68 | + with open(cache_path, mode='wb') as file: |
| 69 | + pickle.dump(obj, file) |
| 70 | + |
| 71 | + print("- Data saved to cache-file: " + cache_path) |
| 72 | + |
| 73 | + return obj |
| 74 | + |
| 75 | + |
| 76 | +######################################################################## |
| 77 | + |
| 78 | + |
| 79 | +def convert_numpy2pickle(in_path, out_path): |
| 80 | + """ |
| 81 | + Convert a numpy-file to pickle-file. |
| 82 | +
|
| 83 | + The first version of the cache-function used numpy for saving the data. |
| 84 | + Instead of re-calculating all the data, you can just convert the |
| 85 | + cache-file using this function. |
| 86 | +
|
| 87 | + :param in_path: |
| 88 | + Input file in numpy-format written using numpy.save(). |
| 89 | +
|
| 90 | + :param out_path: |
| 91 | + Output file written as a pickle-file. |
| 92 | +
|
| 93 | + :return: |
| 94 | + Nothing. |
| 95 | + """ |
| 96 | + |
| 97 | + # Load the data using numpy. |
| 98 | + data = np.load(in_path) |
| 99 | + |
| 100 | + # Save the data using pickle. |
| 101 | + with open(out_path, mode='wb') as file: |
| 102 | + pickle.dump(data, file) |
| 103 | + |
| 104 | + |
| 105 | +######################################################################## |
| 106 | + |
| 107 | +if __name__ == '__main__': |
| 108 | + # This is a short example of using a cache-file. |
| 109 | + |
| 110 | + # This is the function that will only get called if the result |
| 111 | + # is not already saved in the cache-file. This would normally |
| 112 | + # be a function that takes a long time to compute, or if you |
| 113 | + # need persistent data for some other reason. |
| 114 | + def expensive_function(a, b): |
| 115 | + return a * b |
| 116 | + |
| 117 | + print('Computing expensive_function() ...') |
| 118 | + |
| 119 | + # Either load the result from a cache-file if it already exists, |
| 120 | + # otherwise calculate expensive_function(a=123, b=456) and |
| 121 | + # save the result to the cache-file for next time. |
| 122 | + result = cache(cache_path='cache_expensive_function.pkl', |
| 123 | + fn=expensive_function, a=123, b=456) |
| 124 | + |
| 125 | + print('result =', result) |
| 126 | + |
| 127 | + # Newline. |
| 128 | + print() |
| 129 | + |
| 130 | + # This is another example which saves an object to a cache-file. |
| 131 | + |
| 132 | + # We want to cache an object-instance of this class. |
| 133 | + # The motivation is to do an expensive computation only once, |
| 134 | + # or if we need to persist the data for some other reason. |
| 135 | + class ExpensiveClass: |
| 136 | + def __init__(self, c, d): |
| 137 | + self.c = c |
| 138 | + self.d = d |
| 139 | + self.result = c * d |
| 140 | + |
| 141 | + def print_result(self): |
| 142 | + print('c =', self.c) |
| 143 | + print('d =', self.d) |
| 144 | + print('result = c * d =', self.result) |
| 145 | + |
| 146 | + print('Creating object from ExpensiveClass() ...') |
| 147 | + |
| 148 | + # Either load the object from a cache-file if it already exists, |
| 149 | + # otherwise make an object-instance ExpensiveClass(c=123, d=456) |
| 150 | + # and save the object to the cache-file for the next time. |
| 151 | + obj = cache(cache_path='cache_ExpensiveClass.pkl', |
| 152 | + fn=ExpensiveClass, c=123, d=456) |
| 153 | + |
| 154 | + obj.print_result() |
| 155 | + |
| 156 | +######################################################################## |
0 commit comments