merge latest

bcmi · Nov 13, 2023 · 84beba8 · 84beba8
1 parent 9cb4e8b
commit 84beba8
Show file tree

Hide file tree

Showing 329 changed files with 51,256 additions and 37,203 deletions.
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -1,15 +1,15 @@
-version: "2"
-
-build:
-  os: "ubuntu-22.04"
-  tools:
-    python: "3.8"
-
-python:
-  install:
-    - requirements: requirements/docs.txt
-    - requirements: requirements.txt
-    - method: pip
-      path: .
-sphinx:
-  fail_on_warning: true
+version: "2"
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.8"
+
+python:
+  install:
+    - requirements: requirements/docs.txt
+    - requirements: requirements.txt
+    - method: pip
+      path: .
+# sphinx:
+#   fail_on_warning: true
diff --git a/LICENSE b/LICENSE
diff --git a/demo/demo.ipynb b/demo/demo.ipynb
@@ -8,21 +8,24 @@
    "source": [
     "# composite image generation by copy-and-paste followed by image blending.\n",
     "from libcom import get_composite_image\n",
-    "from libcom.utils.process_image import make_image_grid\n",
+    "from libcom.utils.process_image import make_image_grid, draw_bbox_on_image\n",
     "import cv2\n",
-    "bg_img  = '../tests/source/background/8351494f79f6bf8f_m03k3r_73176af9_08.png'\n",
-    "bbox = [223, 104, 431, 481] # x1,y1,x2,y2\n",
-    "fg_img  = '../tests/source/foreground/8351494f79f6bf8f_m03k3r_73176af9_08.png'\n",
-    "fg_mask = '../tests/source/foreground_mask/8351494f79f6bf8f_m03k3r_73176af9_08.png'\n",
-    "# generate composite images by naive methods\n",
-    "img_list = [bg_img, fg_img]\n",
-    "comp_img1, comp_mask1 = get_composite_image(fg_img, fg_mask, bg_img, bbox, 'none')\n",
-    "comp_img2, comp_mask2 = get_composite_image(fg_img, fg_mask, bg_img, bbox, 'gaussian')\n",
-    "comp_img3, comp_mask3 = get_composite_image(fg_img, fg_mask, bg_img, bbox, 'poisson')\n",
-    "img_list += [comp_img1, comp_mask1, comp_img2, comp_mask2, comp_img3, comp_mask3]\n",
-    "# visualization results\n",
-    "grid_img  = make_image_grid(img_list, cols=4)\n",
-    "cv2.imwrite('../docs/_static/image/generatecomposite_result1.jpg', grid_img)"
+    "test_dir = 'source/'\n",
+    "img_list = ['1.jpg', '8.jpg']\n",
+    "bbox_list = [[1000, 895, 1480, 1355], [1170, 944, 2331, 3069]] \n",
+    "for i,img_name in enumerate(img_list):\n",
+    "    bg_img  = test_dir + 'background/' + img_name\n",
+    "    bbox    = bbox_list[i] # x1,y1,x2,y2\n",
+    "    fg_img  = test_dir + 'foreground/' + img_name\n",
+    "    fg_mask = test_dir + 'foreground_mask/' + img_name.replace('.jpg', '.png')\n",
+    "    # generate composite images by naive methods\n",
+    "    comp_img1, comp_mask1 = get_composite_image(fg_img, fg_mask, bg_img, bbox, 'none')\n",
+    "    comp_img2, comp_mask2 = get_composite_image(fg_img, fg_mask, bg_img, bbox, 'gaussian')\n",
+    "    comp_img3, comp_mask3 = get_composite_image(fg_img, fg_mask, bg_img, bbox, 'poisson')\n",
+    "    vis_list = [bg_img, fg_img, comp_img1, comp_mask1, comp_img2, comp_mask2, comp_img3, comp_mask3]\n",
+    "    # visualization results\n",
+    "    grid_img  = make_image_grid(vis_list, cols=4)\n",
+    "    cv2.imwrite(f'../docs/_static/image/generatecomposite_result{i+1}.jpg', grid_img)"
    ]
   },
   {
@@ -31,15 +34,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# transfer foreground color to fit background scene.\n",
+    "# color-transfer: transfer foreground color to fit background scene.\n",
     "from libcom import color_transfer\n",
     "from libcom.utils.process_image import make_image_grid\n",
     "import cv2\n",
-    "comp_img  = '../tests/source/composite/5b9e0751ca458cad_m0gjkl_40736b77_04.png'\n",
-    "comp_mask = '../tests/source/composite_mask/5b9e0751ca458cad_m0gjkl_40736b77_04.png'\n",
-    "trans_img = color_transfer(comp_img, comp_mask)\n",
+    "comp_img1  = '../tests/source/composite/1.jpg'\n",
+    "comp_mask1 = '../tests/source/composite_mask/1.png'\n",
+    "trans_img1 = color_transfer(comp_img1, comp_mask1)\n",
+    "comp_img2  = '../tests/source/composite/8.jpg'\n",
+    "comp_mask2 = '../tests/source/composite_mask/8.png'\n",
+    "trans_img2 = color_transfer(comp_img2, comp_mask2)\n",
     "# visualization results\n",
-    "grid_img  = make_image_grid([comp_img, comp_mask, trans_img])\n",
+    "grid_img  = make_image_grid([comp_img1, comp_mask1, trans_img1, \n",
+    "                            comp_img2, comp_mask2, trans_img2], cols=3)\n",
     "cv2.imwrite('../docs/_static/image/colortransfer_result1.jpg', grid_img)"
    ]
   },
@@ -54,10 +61,16 @@
     "from libcom.utils.process_image import make_image_grid\n",
     "import cv2\n",
     "net = HarmonyScoreModel(device=0, model_type='BargainNet')\n",
-    "comp_img  = '../tests/source/composite/1bb218825e370c51_m02p5f1q_4a9db762_30.png'\n",
-    "comp_mask = '../tests/source/composite_mask/1bb218825e370c51_m02p5f1q_4a9db762_30.png'\n",
-    "score     = net(comp_img, comp_mask)\n",
-    "grid_img  = make_image_grid([comp_img, comp_mask], text_list=[f'harmony_score:{score:.2f}', 'composite-mask'])\n",
+    "test_dir   = '../tests/harmony_score_prediction/'\n",
+    "img_names  = ['vaulted-cellar-247391_inharm.jpg', 'ameland-5651866_harm.jpg']\n",
+    "vis_list,scores = [], []\n",
+    "for img_name in img_names:\n",
+    "    comp_img  = test_dir + 'composite/' + img_name\n",
+    "    comp_mask = test_dir + 'composite_mask/' + img_name\n",
+    "    score     = net(comp_img, comp_mask)\n",
+    "    vis_list += [comp_img, comp_mask]\n",
+    "    scores.append(score)\n",
+    "grid_img  = make_image_grid(vis_list, text_list=[f'harmony_score:{scores[0]:.2f}', 'composite-mask', f'harmony_score:{scores[1]:.2f}', 'composite-mask'])\n",
     "cv2.imwrite('../docs/_static/image/harmonyscore_result1.jpg', grid_img)"
    ]
   },
@@ -69,13 +82,20 @@
    "source": [
     "# Object Placement Assessment (OPA) score prediction\n",
     "from libcom import OPAScoreModel\n",
+    "from libcom import get_composite_image\n",
     "from libcom.utils.process_image import make_image_grid\n",
     "import cv2\n",
     "net = OPAScoreModel(device=0, model_type='SimOPA')\n",
-    "comp_img  = '../tests/source/composite/1bb218825e370c51_m02p5f1q_4a9db762_30.png'\n",
-    "comp_mask = '../tests/source/composite_mask/1bb218825e370c51_m02p5f1q_4a9db762_30.png'\n",
-    "score = net(comp_img, comp_mask)\n",
-    "grid_img  = make_image_grid([comp_img, comp_mask], text_list=[f'opa_score:{score:.2f}', 'composite-mask'])\n",
+    "test_dir  = './source'\n",
+    "bg_img    = 'source/background/17.jpg'\n",
+    "fg_img    = 'source/foreground/17.jpg'\n",
+    "fg_mask   = 'source/foreground_mask/17.png'\n",
+    "bbox_list = [[475, 697, 1275, 1401], [475, 300, 1275, 1004]]\n",
+    "comp1, comp_mask1 = get_composite_image(fg_img, fg_mask, bg_img, bbox_list[0])\n",
+    "comp2, comp_mask2 = get_composite_image(fg_img, fg_mask, bg_img, bbox_list[1])\n",
+    "score1 = net(comp1, comp_mask1)\n",
+    "score2 = net(comp2, comp_mask2)\n",
+    "grid_img  = make_image_grid([comp1, comp_mask1, comp2, comp_mask2], text_list=[f'opa_score:{score1:.2f}', 'composite-mask', f'opa_score:{score2:.2f}', 'composite-mask'])\n",
     "cv2.imwrite('../docs/_static/image/opascore_result1.jpg', grid_img)"
    ]
   },
@@ -85,19 +105,22 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# fos_score demo\n",
+    "# Foreground Object Search (FOS) score prediction\n",
     "from libcom.utils.process_image import make_image_grid\n",
     "from libcom import FOSScoreModel\n",
     "import cv2\n",
+    "import torch\n",
+    "\n",
+    "task_name = 'fos_score_prediction'\n",
     "MODEL_TYPE = 'FOS_D' # choose from 'FOS_D', 'FOS_E'\n",
     "background = '../tests/source/background/f80eda2459853824_m09g1w_b2413ec8_11.png'\n",
     "fg_bbox    = [175, 82, 309, 310] # x1,y1,x2,y2\n",
     "foreground = '../tests/source/foreground/f80eda2459853824_m09g1w_b2413ec8_11.png'\n",
     "foreground_mask = '../tests/source/foreground_mask/f80eda2459853824_m09g1w_b2413ec8_11.png'\n",
-    "net        = FOSScoreModel(device=0, model_type=MODEL_TYPE)\n",
-    "score      = net(background_image, foreground_image, bounding_box, foreground_mask=foreground_mask)\n",
-    "grid_img   = make_image_grid([background_image, foreground_image, composite_image], text_list=[f'fos_score:{score:.2f}'])\n",
-    "cv2.imshow('fos_score_demo', grid_img)"
+    "net = FOSScoreModel(device=0, model_type=MODEL_TYPE)\n",
+    "score = net(background_image, foreground_image, bounding_box, foreground_mask=foreground_mask)\n",
+    "grid_img  = make_image_grid([background_image, foreground_image, composite_image], text_list=[f'fos_score:{score:.2f}'])\n",
+    "cv2.imshow('fos_score_demo', grid_img)\n"
    ]
   },
   {
@@ -108,16 +131,21 @@
    "source": [
     "# controllable image composition\n",
     "from libcom import ControlComModel\n",
-    "from libcom.utils.process_image import make_image_grid\n",
+    "from libcom.utils.process_image import make_image_grid, draw_bbox_on_image\n",
     "import cv2\n",
-    "bg_img   = '../tests/source/background/f80eda2459853824_m09g1w_b2413ec8_11.png'\n",
-    "fg_bbox  = [175, 82, 309, 310] # x1,y1,x2,y2\n",
-    "fg_img   = '../tests/source/foreground/f80eda2459853824_m09g1w_b2413ec8_11.png'\n",
-    "fg_mask  = '../tests/source/foreground_mask/f80eda2459853824_m09g1w_b2413ec8_11.png'\n",
-    "net      = ControlComModel(device=0, model_type='ControlCom')\n",
-    "comp_img = net(bg_img, fg_img, fg_bbox, fg_mask, task=['blending', 'harmonization'])\n",
-    "grid_img = make_image_grid([bg_img, fg_img, comp_img[0], comp_img[1]])\n",
-    "cv2.imwrite('../docs/_static/image/controlcom_result1.jpg', grid_img)"
+    "img_names = ['6c5601278dcb5e6d_m09728_f5cd2891_17.png', '000000460450.png']\n",
+    "bboxes    = [[130, 91, 392, 271], [134, 158, 399, 511]] # x1,y1,x2,y2\n",
+    "test_dir  = '../tests/controllable_composition/'\n",
+    "for i in range(len(img_names)):\n",
+    "    bg_img  = test_dir + 'background/' + img_names[i]\n",
+    "    fg_img  = test_dir + 'foreground/' + img_names[i]\n",
+    "    bbox    = bboxes[i]\n",
+    "    mask    = test_dir + 'foreground_mask/' + img_names[i]\n",
+    "    net     = ControlComModel(device=0)\n",
+    "    comp    = net(bg_img, fg_img, bbox, mask, task=['blending', 'harmonization'])\n",
+    "    bg_img  = draw_bbox_on_image(bg_img, bbox)\n",
+    "    grid_img = make_image_grid([bg_img, fg_img, comp[0], comp[1]])\n",
+    "    cv2.imwrite('../docs/_static/image/controlcom_result{}.jpg'.format(i+1), grid_img)\n"
    ]
   }
  ],

diff --git a/docs/Makefile b/docs/Makefile
@@ -1,20 +1,20 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/_static/css/readthedocs.css b/docs/_static/css/readthedocs.css
@@ -1,6 +1,6 @@
-.header-logo {
-    background-image: url("../image/LOGO.png");
-    background-size: 60px 60px;
-    height: 60px;
-    width: 60px;
-}
+.header-logo {
+    background-image: url("../image/LOGO.png");
+    background-size: 60px 60px;
+    height: 60px;
+    width: 60px;
+}
diff --git a/docs/_static/image/colortransfer_result1.jpg b/docs/_static/image/colortransfer_result1.jpg
diff --git a/docs/_static/image/controlcom_result1.jpg b/docs/_static/image/controlcom_result1.jpg
diff --git a/docs/_static/image/controlcom_result2.jpg b/docs/_static/image/controlcom_result2.jpg
diff --git a/docs/_static/image/fopa_heatmap_FOPA_result1.png b/docs/_static/image/fopa_heatmap_FOPA_result1.png
diff --git a/docs/_static/image/fopa_heatmap_FOPA_result3.png b/docs/_static/image/fopa_heatmap_FOPA_result3.png
diff --git a/docs/_static/image/fos_score_result2.jpg b/docs/_static/image/fos_score_result2.jpg
diff --git a/docs/_static/image/fos_score_result3.jpg b/docs/_static/image/fos_score_result3.jpg
diff --git a/docs/_static/image/generatecomposite_result1.jpg b/docs/_static/image/generatecomposite_result1.jpg
diff --git a/docs/_static/image/generatecomposite_result2.jpg b/docs/_static/image/generatecomposite_result2.jpg
diff --git a/docs/_static/image/harmonyscore_result1.jpg b/docs/_static/image/harmonyscore_result1.jpg
diff --git a/docs/_static/image/image_harmonization_CDT_result1.jpg b/docs/_static/image/image_harmonization_CDT_result1.jpg
diff --git a/docs/_static/image/image_harmonization_CDT_result2.jpg b/docs/_static/image/image_harmonization_CDT_result2.jpg
diff --git a/docs/_static/image/image_harmonization_PCT_result1.jpg b/docs/_static/image/image_harmonization_PCT_result1.jpg
diff --git a/docs/_static/image/image_harmonization_PCT_result2.jpg b/docs/_static/image/image_harmonization_PCT_result2.jpg
diff --git a/docs/_static/image/inharmonious_localization_result3.jpg b/docs/_static/image/inharmonious_localization_result3.jpg
diff --git a/docs/_static/image/inharmonious_localization_result3_4.jpg b/docs/_static/image/inharmonious_localization_result3_4.jpg
diff --git a/docs/_static/image/inharmonious_localization_result4.jpg b/docs/_static/image/inharmonious_localization_result4.jpg
diff --git a/docs/_static/image/opascore_result1.jpg b/docs/_static/image/opascore_result1.jpg
diff --git a/docs/_static/image/painterly_image_harmonization_result1.jpg b/docs/_static/image/painterly_image_harmonization_result1.jpg
diff --git a/docs/_static/image/painterly_image_harmonization_result2.jpg b/docs/_static/image/painterly_image_harmonization_result2.jpg
diff --git a/docs/_static/image/painterly_image_harmonization_result3.jpg b/docs/_static/image/painterly_image_harmonization_result3.jpg
diff --git a/docs/_static/image/shadow_generation_result3.jpg b/docs/_static/image/shadow_generation_result3.jpg
diff --git a/docs/_static/image/shadow_generation_result4.jpg b/docs/_static/image/shadow_generation_result4.jpg
diff --git a/docs/api.rst b/docs/api.rst
@@ -1,5 +1,63 @@
-libcom.fos_score
--------------------
-.. automodule:: libcom.fos_score
-    :members:
+libcom.color_transfer
+----------------------
+.. automodule:: libcom.color_transfer
+    :members:
+
+libcom.controllable_composition
+--------------------------------
+.. automodule:: libcom.controllable_composition
+    :members:
+    :special-members: __call__
+
+libcom.fos_score
+-------------------
+.. automodule:: libcom.fos_score
+    :members:
+    :special-members: __call__
+
+libcom.harmony_score
+-----------------------
+.. automodule:: libcom.harmony_score
+    :members:
+    :special-members: __call__
+
+libcom.naive_composition
+-------------------------
+.. automodule:: libcom.naive_composition
+    :members:
+
+libcom.opa_score
+------------------
+.. automodule:: libcom.opa_score
+    :members:
+    :special-members: __call__
+
+libcom.image_harmonization
+---------------------------
+.. automodule:: libcom.image_harmonization
+    :members:
+    :special-members: __call__
+
+libcom.inharmonious_region_localization
+----------------------------------------
+.. automodule:: libcom.inharmonious_region_localization
+    :members:
+    :special-members: __call__
+
+libcom.painterly_image_harmonization
+----------------------------------------
+.. automodule:: libcom.painterly_image_harmonization
+    :members:
+    :special-members: __call__
+
+libcom.fopa_heat_map
+----------------------------------------
+.. automodule:: libcom.fopa_heat_map
+    :members:
+    :special-members: __call__
+
+libcom.shadow_generation
+----------------------------------------
+.. automodule:: libcom.shadow_generation
+    :members:
     :special-members: __call__
diff --git a/docs/color_transfer.md b/docs/color_transfer.md
@@ -1,14 +1,12 @@
-# Color Transfer
-
-Given a composite image obtained by copy-and-paste, we transfer the color of foreground object to fit background scene by using:
-
-**Reinhard's algorithm**:
-
-> **Color Transfer between Images**  [[paper]](https://www.cs.tau.ac.il/~turkel/imagepapers/ColorTransfer.pdf)<br>
->
-> E. Reinhard, M. Adhikhmin, B. Gooch, P. Shirley <br>
-> Accepted by **IEEE Computer Graphics and Applications 2001**.
-
-## Brief Method Summary
-
-Reinhard's algorithm achieves color transfer by taking background as source image and applying its color characteristic to foreground object. Specifically,  this method adjusts the mean and the standard deviation of L*αβ* channels to match the global color distribution of two images. 
+# Color Transfer
+
+Given a composite image obtained by cut-and-paste, we can adjust the foreground color to match the background by using Reinhard's algorithm, which is a traditional color transfer method. When the requirement for the harmonized result is not very high, we can use [traditional color transfer methods](https://github.com/bcmi/Color-Transfer-for-Image-Harmonization) instead of image harmonization methods. When the foreground and background have pure colors and we simply want to match their colors, traditional color transfer methods may work better than image harmonization methods. 
+
+> **Color Transfer between Images**  [[paper]](https://www.cs.tau.ac.il/~turkel/imagepapers/ColorTransfer.pdf)<br>
+>
+> E. Reinhard, M. Adhikhmin, B. Gooch, P. Shirley <br>
+> Accepted by **IEEE Computer Graphics and Applications 2001**.
+
+## Brief Method Summary
+
+Reinhard's algorithm achieves color transfer by taking background as source and applying its color characteristic to foreground object. Specifically,  this method adjusts the mean and the standard deviation of L*αβ* channels to match the color distributions between foreground and background.