update app

NN/utils.py

@@ -2,6 +2,23 @@
 import tensorflow_addons as tfa
 import tensorflow_probability as tfp
 import tensorflow.keras.layers as L
+import tensorflow as tf
+import tensorflow_probability as tfp
+
+def gaussian_kernel(size, stdsPx):
+  stds = tf.cast(stdsPx, tf.float32) / size
+  B = tf.shape(stds)[0]
+  stds = tf.reshape(stds, [B, 1])
+  x = tf.linspace(-size // 2 + 1, size // 2 + 1, size)
+  x = tf.cast(x ** 2, tf.float32)
+  x = tf.tile(x[None], [B, 1])
+  x = tf.nn.softmax(-x / (2.0 * (stds**2)))
+  x = tf.matmul(x[:, :, None], x[:, None, :])
+  gauss = tf.reshape(x, [B, size, size, 1])
+  gauss = tf.repeat(gauss, 3, axis=-1)
+  gauss = tf.repeat(gauss, 3, axis=0)
+  gauss = tf.transpose(gauss, [1, 2, 3, 0]) # [B, size, size, 1] => [size, size, 1, B]
+  return gauss
 
 def masked(x, mask):
   '''
@@ -139,4 +156,21 @@ def is_namedtuple(obj) -> bool:
     isinstance(obj, tuple) and
     hasattr(obj, '_asdict') and
     hasattr(obj, '_fields')
-  )
+  )
+
+if '__main__' == __name__:
+  print('Utils test')
+  print('All tests passed successfully!')
+  import cv2
+  import tensorflow_addons as tfa
+
+  img = cv2.imread('d:\photo_2024-03-26_15-48-33.jpg') 
+  img = img.astype('float32') / 255.0
+  gaussians = gaussian_kernel(48, tf.constant([10., 20., 30.]))
+  imgG = tf.nn.conv2d(img[None], gaussians, strides=[1, 1, 1, 1], padding='SAME')[0]
+
+  for i in [3, 6, 9]:
+    g = imgG[..., i-3:i].numpy()
+    g = (g * 255.0).astype('uint8')
+    cv2.imshow('Gaussian %i' % i, g)
+    cv2.waitKey(0)

huggingface/HF/NN/CARDirectionModelProxy.py

@@ -6,6 +6,10 @@ def __init__(self, model):
     self._model = model
     return
 
+  @property
+  def processor(self):
+    return self._model.processor
+
   def __call__(self,
     threshold, start, end, steps, decay, noiseStddev, convergeThreshold,
     reverseArgs={},

huggingface/HF/NN/CInterpolantVisualization.py

@@ -7,6 +7,8 @@
 from mpl_toolkits.mplot3d import Axes3D
 from NN.restorators.samplers.CSamplerWatcher import CSamplerWatcher
 from NN.utils import extractInterpolated, ensure4d
+import NN.utils as NNU
+from HF.Utils import toPILImage
 
 from collections import namedtuple
 CCollectedSteps = namedtuple('CCollectedSteps', ['value', 'x0', 'x1', 'totalSteps', 'totalPoints'])
@@ -124,6 +126,10 @@ def __init__(self, model):
     self._model = model
     return
 
+  @property
+  def processor(self):
+    return self._model.processor
+
   def _generateFilename(self, **kwargs):
     folder = os.path.abspath('tmp')
     os.makedirs(folder, exist_ok=True)
@@ -143,7 +149,7 @@ def _generateFilename(self, **kwargs):
   def _collectSteps(self, points, initialValues, kwargs):
     NSamples = len(points)
     watcher = CSamplerWatcher(
-      steps=1000, # 1000 steps at most
+      steps=100, # 100 steps at most
       tracked=dict(
         value=(NSamples, 3),
         x0=(NSamples, 3),
@@ -158,7 +164,7 @@ def _collectSteps(self, points, initialValues, kwargs):
       initialValues=initialValues,
       reverseArgs=dict(
         **reverseArgs,
-        algorithmInterceptor=watcher,
+        algorithmInterceptor=watcher
       ),
     )
     N = watcher.iteration + 1
@@ -238,6 +244,33 @@ def _generateVideo(self, writer, originalColors, collectedSteps):
       plt.close(fig)
     return
 
+  def _generateProcessVideo(self, writer, resolution, collectedSteps, name):
+    FIG_COLS = 1
+    FIG_ROWS = 1
+    fig = plt.figure(figsize=(FIG_COLS * 9, FIG_ROWS * 6))
+    gs = fig.add_gridspec(FIG_ROWS, FIG_COLS)
+    axs = [
+      fig.add_subplot(gs[0, 0]), # image
+    ]
+    try:
+      processor = self.processor
+      for step in range(collectedSteps.totalSteps):
+        fig.suptitle(f'Step {step+1}/{collectedSteps.totalSteps}')
+        # plot the image
+        image = collectedSteps.value[step] if name == 'value' else collectedSteps.x0[step]
+        image = image.reshape(resolution, resolution, 3)
+        image = processor.range.convertBack(image)
+        image = toPILImage(image, isBGR=False)
+        axs[0].imshow(image)
+
+        fig.tight_layout()
+        fig.show()
+        writer.append_data(plot2image(fig))
+        continue
+    finally:
+      plt.close(fig)
+    return
+
   def _extractOriginalColors(self, points, raw, **kwargs):
     assert raw.ndim == 3, 'Unexpected shape of raw'
     assert raw.shape[-1] == 3, 'Unexpected number of channels in raw'
@@ -260,18 +293,14 @@ def _initialValuesFor(self, initialValues, seed, N):
       return np.random.RandomState(seed).uniform(size=(1, N, 3)).astype(np.float32)
 
     raise ValueError(f'Unexpected initialValues: {initialValues}')
-  
-  def __call__(self,
-    VT_numPoints=None, VT_fps=None, VT_useOriginalColors=None, VT_seed=-1,
-    VT_initialValues=None,
+
+  def _trajectories(self,
+    VT_numPoints, VT_fps, VT_seed, VT_initialValues,
+    VT_useOriginalColors, 
     **kwargs
   ):
-    if VT_numPoints is None: return self._model(**kwargs)
     VT_numPoints = int(VT_numPoints)
-    VT_seed = None if VT_seed < 0 else int(VT_seed)
-    VT_fps = int(VT_fps)
     VT_useOriginalColors = bool(VT_useOriginalColors)
-
     videoFileName = self._generateFilename(
       VT_numPoints=VT_numPoints, VT_seed=VT_seed, VT_fps=VT_fps,
       VT_initialValues=VT_initialValues,
@@ -298,6 +327,57 @@ def __call__(self,
       )
     return { 'video': videoFileName, }
 
+  def _process(self,
+    VT_fps, VT_seed, VT_initialValues, VT_resolution, VT_show,
+    **kwargs
+  ):
+    assert VT_show in ['value', 'x0'], f'Unexpected VT_show: {VT_show}'
+    VT_resolution = int(VT_resolution)
+    assert 64 <= VT_resolution <= 512, 'Unexpected resolution'
+    videoFileName = self._generateFilename(
+      VT_fps=VT_fps, VT_seed=VT_seed, VT_initialValues=VT_initialValues, 
+      VT_resolution=VT_resolution,
+      **kwargs
+    )
+
+    with imageio.get_writer(videoFileName, mode='I', fps=VT_fps) as writer:
+      self._generateProcessVideo(
+        writer=writer,
+        resolution=VT_resolution,
+        collectedSteps=self._collectSteps(
+          points=NNU.generateSquareGrid(size=VT_resolution, scale=1.0, shift=0.0),
+          initialValues=self._initialValuesFor(
+            initialValues=VT_initialValues, seed=VT_seed, N=VT_resolution ** 2
+          ),
+          kwargs=kwargs,
+        ),
+        name=VT_show,
+      )
+    return { 'video': videoFileName, }
+
+  def __call__(self,
+    VT_fps=None, VT_seed=-1,
+    VT_kind=None,
+    **kwargs
+  ):
+    if VT_kind is None: return self._model(**kwargs)
+    VT_seed = None if VT_seed < 0 else int(VT_seed)
+    VT_fps = int(VT_fps)
+    assert VT_kind in ['trajectories', 'process'], f'Unexpected VT_kind: {VT_kind}'
+
+    if VT_kind == 'trajectories':
+      return self._trajectories(
+        VT_fps=VT_fps, VT_seed=VT_seed,
+        **kwargs
+      )
+    if VT_kind == 'process':
+      return self._process(
+        VT_fps=VT_fps, VT_seed=VT_seed,
+        **kwargs
+      )
+
+    raise ValueError(f'Unexpected VT_kind: {VT_kind}')
+
   @property
   def kind(self): return self._model.kind
 

huggingface/HF/NN/CWithAblations.py

@@ -5,6 +5,10 @@ def __init__(self, model):
     self._model = model
     return
 
+  @property
+  def processor(self):
+    return self._model.processor
+
   def __call__(self, randomizePositions, encoderContext, encoderIntermediate, reverseArgs={}, **kwargs):
     encoderContextMappping = {
       'Both': (False, False),

huggingface/HF/NN/CWithDDIM.py

@@ -5,6 +5,10 @@ def __init__(self, model):
     self._model = model
     return
 
+  @property
+  def processor(self):
+    return self._model.processor
+
   def __call__(self, 
     DDIM_stochasticity, DDIM_K, DDIM_clipping, DDIM_projectNoise, DDIM_noiseStddev,
     reverseArgs={}, **kwargs

huggingface/HF/UI/areas/DDIMParametersArea.py

@@ -1,56 +1,20 @@
 import gradio as gr
 from ..common import noiseProviderStddev
 
-def _presets(extraPresets, defaultPreset):
-  # (stochasticity, K, clipping, projectNoise, noiseStddev)
-  presets = {
-    **extraPresets,
-    'DDPM': (1.0, 1, False, False, 'normal'),
-    'DDIM': (0.0, 1, False, False, 'normal'),
-  }
-  def onSelect(presetName):
-    preset = presets.get(presetName, None)
-    assert preset is not None, f'Unknown preset: {presetName}'
-    return preset
-
-  names = list(presets.keys())
-  defaultPreset = defaultPreset if defaultPreset in names else names[0]
-  defaultPresetValues = presets[defaultPreset]
-  return names, onSelect, {
-    'name': defaultPreset,
-    'stochasticity': defaultPresetValues[0],
-    'K': defaultPresetValues[1],
-    'clipping': defaultPresetValues[2],
-    'projectNoise': defaultPresetValues[3],
-    'noiseStddev': defaultPresetValues[4],
-  }
-
-def DDIMParametersArea(extraPresets={}, defaultPreset=None):
-  presets, onSelectPreset, defaultPreset = _presets(extraPresets, defaultPreset=defaultPreset)
+def DDIMParametersArea():
   with gr.Group():
-    preset = gr.Dropdown(
-      choices=presets, value=presets[0], label='Parameters preset',
-      interactive=True, allow_custom_value=False
-    )
     # DDIM parameters
     stochasticity = gr.Slider(
       minimum=0.0, maximum=1.0, step=0.05, label='Stochasticity', interactive=True,
-      value=defaultPreset['stochasticity']
+      value=1.0
     )
     K = gr.Slider(
       minimum=1, maximum=30, step=1, label='K', interactive=True,
-      value=defaultPreset['K']
-    )
-    clipping = gr.Checkbox(label='Clipping to [-1, 1]', interactive=True, value=defaultPreset['clipping'])
-    projectNoise = gr.Checkbox(label='Use noise projection', interactive=True, value=defaultPreset['projectNoise'])
-    noiseStddev = noiseProviderStddev(defaultPreset['noiseStddev'])
-
-    # bind preset to parameters
-    preset.change(
-      onSelectPreset,
-      inputs=[preset],
-      outputs=[stochasticity, K, clipping, projectNoise, noiseStddev],
+      value=10
     )
+    clipping = gr.Checkbox(label='Clipping to [-1, 1]', interactive=True, value=False)
+    projectNoise = gr.Checkbox(label='Use noise projection', interactive=True, value=True)
+    noiseStddev = noiseProviderStddev('normal')
 
   return dict(
     DDIM_stochasticity=stochasticity,

huggingface/HF/UI/areas/visualizeArea.py

@@ -0,0 +1,77 @@
+import gradio as gr
+
+def trajectoriesArea(submit, parameters):
+  with gr.Tab(label='Trajectories'):
+    gr.Markdown('''
+    # Trajectories
+    Here you can visualize the trajectories of the particles.
+    ''')
+    # number of points, button "Visualize", area for video
+    numPoints = gr.Slider(
+      label='Number of sampled points',
+      value=10, minimum=1, maximum=200, step=1
+    )
+    useOriginalColors = gr.Checkbox(label='Show original colors', value=False)
+    button = gr.Button(value='Visualize')
+    video = gr.Video(label='Video', interactive=False)
+    # Hidden kind parameter
+    kind = gr.Textbox(label='Kind', value='trajectories', visible=False)
+    # bind button
+    submit(
+      btn=button,
+      VT_kind=kind,
+      VT_numPoints=numPoints,
+      VT_useOriginalColors=useOriginalColors,
+      **parameters,
+      outputs={'video': video}
+    )
+
+def imageArea(submit, parameters):
+  with gr.Tab(label='Image Restoration Process'):
+    gr.Markdown('''
+    # Images
+    Here you can visualize the image restoration process step by step.
+    ''')
+    VT_resolution = gr.Slider(
+      label='Resolution', value=64, minimum=8, maximum=512, step=1
+    )
+    # Radio buttons for the displayed values. 'value' or 'x0
+    VT_show = gr.Radio(
+      label='Show',
+      choices=['value', 'x0'], value='value',
+    )
+    button = gr.Button(value='Visualize')
+    video = gr.Video(label='Video', interactive=False)
+    # Hidden kind parameter
+    kind = gr.Textbox(label='Kind', value='process', visible=False)
+    # bind button
+    submit(
+      btn=button,
+      VT_kind=kind,
+      VT_resolution=VT_resolution,
+      VT_show=VT_show,
+      **parameters,
+      outputs={'video': video}
+    )
+
+def visualizeArea(submit):
+  with gr.Accordion('Visualize Area', open=False):
+    fps = gr.Slider(label='Frames per second', value=2, minimum=1, maximum=30, step=1)
+    VT_seed = gr.Number(
+      label='Seed (-1 for random)', value=-1, 
+      minimum=-1, maximum=1000000, step=1, interactive=True
+    )
+    VT_initialValues = gr.Radio(
+      label='Initial values',
+      choices=['Seeded', 'Zeros'], value='Seeded',
+    )
+
+    parameters = dict(
+      VT_fps=fps,
+      VT_initialValues=VT_initialValues,
+      VT_seed=VT_seed,
+    )
+    trajectoriesArea(submit, parameters)
+    imageArea(submit, parameters)
+    pass
+  return