DOC add Mandrill example (#10)

Co-authored-by: Hande Gözükan <[email protected]>
Co-authored-by: Romain Primet <[email protected]>

.github/workflows/deploy_ghpages.yml

@@ -10,12 +10,15 @@ jobs:
       - uses: actions/checkout@v2
       - name: Generate HTML docs
         uses: ammaraskar/sphinx-action@master
+        env:
+          ALLOW: --allow-run-as-root
         with:
           docs-folder: "docs/"
           pre-build-command: |
             apt-get update
             apt-get install -y libopenmpi-dev openmpi-bin
             pip install -e .[doc]
+            echo "localhost slots=50">hostfile
       - name:  Upload generated HTML as artifact
         uses: actions/upload-artifact@v2
         with:
@@ -51,5 +54,4 @@ jobs:
           directory: gh-pages
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
-
-
+

.github/workflows/unittests.yml

@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     defaults:
       run:
         shell: bash -l {0}

benchmarks/compare_cdl.py

@@ -1,10 +1,9 @@
-
 import pandas
 import numpy as np
 import matplotlib.pyplot as plt
 from collections import namedtuple
 
-from dicodile.dicodile import dicodile
+from dicodile import dicodile
 from dicodile.data.images import get_hubble
 from dicodile.utils.viz import median_curve
 from dicodile.utils.dictionary import get_lambda_max
@@ -50,7 +49,7 @@ def run_one(method, n_atoms, atom_support, reg, z_positive, n_workers, n_iter,
             'CBPDN': {'rho': 50.0*reg_ + 0.5,
                       'NonNegCoef': z_positive},
             'DictSize': D_init_.shape,
-            }
+        }
         opt = ConvBPDNDictLearn_Consensus.Options(options)
         cdl = ConvBPDNDictLearn_Consensus(
             D_init_, X_, lmbda=reg_, nproc=n_workers, opt=opt, dimK=1, dimN=2)

benchmarks/dicodile_hubble.py

@@ -2,7 +2,7 @@
 import numpy as np
 from scipy import sparse
 
-from dicodile.dicodile import dicodile
+from dicodile import dicodile
 from dicodile.data.images import get_hubble
 from dicodile.utils.viz import plot_atom_and_coefs
 

benchmarks/dicodile_text.py

@@ -13,7 +13,7 @@
 from sklearn.feature_extraction.image import extract_patches_2d
 
 # Import for CDL
-from dicodile.dicodile import dicodile
+from dicodile import dicodile
 
 # Import to initiate the dictionary
 from dicodile.utils.dictionary import prox_d

benchmarks/scaling_2d.py

@@ -15,7 +15,7 @@
 from benchmarks.parallel_resource_balance import ParallelResourceBalance
 
 from dicodile.update_z.dicod import dicod
-from dicodile.data.images import get_mandril
+from dicodile.data.images import fetch_mandrill
 from dicodile.utils import check_random_state
 from dicodile.utils.dictionary import get_lambda_max
 from dicodile.utils.dictionary import init_dictionary
@@ -78,7 +78,7 @@ def run_one_scaling_2d(n_atoms, atom_support, reg, n_workers, strategy, tol,
     # Generate a problem
     print(colorify(79*"=" + f"\n{tag} Start with {n_workers} workers\n" +
                    79*"="))
-    X = get_mandril()
+    X = fetch_mandrill()
     D = init_dictionary(X, n_atoms, atom_support, random_state=random_state)
     reg_ = reg * get_lambda_max(X, D).max()
 

benchmarks/scaling_grid.py

@@ -15,7 +15,7 @@
 from benchmarks.parallel_resource_balance import ParallelResourceBalance
 
 from dicodile.update_z.dicod import dicod
-from dicodile.data.images import get_mandril
+from dicodile.data.images import fetch_mandrill
 from dicodile.utils import check_random_state
 from dicodile.utils.dictionary import get_lambda_max
 from dicodile.utils.dictionary import init_dictionary
@@ -80,7 +80,7 @@ def run_one_grid(n_atoms, atom_support, reg, n_workers, grid, tol,
     # Generate a problem
     print(colorify(79*"=" + f"\n{tag} Start with {n_workers} workers\n" +
                    79*"="))
-    X = get_mandril()
+    X = fetch_mandrill()
     D = init_dictionary(X, n_atoms, atom_support, random_state=random_state)
     reg_ = reg * get_lambda_max(X, D).max()
 

benchmarks/soft_lock.py

@@ -4,7 +4,7 @@
 
 
 from dicodile.update_z.dicod import dicod
-from dicodile.data.images import get_mandril
+from dicodile.data.images import fetch_mandrill
 from dicodile.utils.segmentation import Segmentation
 from dicodile.utils.dictionary import get_lambda_max
 from dicodile.utils.dictionary import init_dictionary
@@ -19,7 +19,7 @@ def run_without_soft_lock(n_atoms=25, atom_support=(12, 12), reg=.01,
                           tol=5e-2, n_workers=100, random_state=60):
     rng = np.random.RandomState(random_state)
 
-    X = get_mandril()
+    X = fetch_mandrill()
     D_init = init_dictionary(X, n_atoms, atom_support, random_state=rng)
     lmbd_max = get_lambda_max(X, D_init).max()
     reg_ = reg * lmbd_max

dicodile/__init__.py

@@ -0,0 +1,3 @@
+from ._dicodile import dicodile
+
+__all__ = ['dicodile']

dicodile/dicodile.py → dicodile/_dicodile.py

@@ -1,4 +1,3 @@
-
 import time
 import numpy as np
 

dicodile/data/images.py

@@ -5,16 +5,16 @@
 from .home import DATA_HOME
 
 
-def get_mandril():
+def fetch_mandrill():
 
-    mandril_dir = DATA_HOME / "images" / "standard_images"
-    mandril_dir.mkdir(parents=True, exist_ok=True)
-    mandril = download(
+    mandrill_dir = DATA_HOME / "images" / "standard_images"
+    mandrill_dir.mkdir(parents=True, exist_ok=True)
+    mandrill = download(
         "http://sipi.usc.edu/database/download.php?vol=misc&img=4.2.03",
-        mandril_dir / "mandril_color.tif"
+        mandrill_dir / "mandrill_color.tif"
     )
 
-    X = plt.imread(mandril) / 255
+    X = plt.imread(mandrill) / 255
     return X.swapaxes(0, 2)
 
 

dicodile/data/tests/test_mandrill.py

@@ -0,0 +1,6 @@
+from dicodile.data.images import fetch_mandrill
+
+
+def test_fetch_mandrill():
+    data = fetch_mandrill()
+    assert(3, 512, 512) == data.shape

dicodile/tests/test_dicodile.py

@@ -1,7 +1,6 @@
-from dicodile.dicodile import dicodile
+from dicodile import dicodile
 from dicodile.data.simulate import simulate_data
 
-
 from dicodile.utils.testing import is_deacreasing
 
 
@@ -11,6 +10,6 @@ def test_dicodile():
                             n_channels=3, noise_level=1e-5, random_state=42)
 
     pobj, times, D_hat, z_hat = dicodile(
-            X, D, reg=.1, z_positive=True, n_iter=10, eps=1e-4,
-            n_workers=1, verbose=2, tol=1e-10)
+        X, D, reg=.1, z_positive=True, n_iter=10, eps=1e-4,
+        n_workers=1, verbose=2, tol=1e-10)
     assert is_deacreasing(pobj)

dicodile/utils/viz.py

@@ -102,7 +102,7 @@ def display_dictionaries(*list_D, styles=None, axes=None, filename=None):
     n_dict = len(list_D)
     D_0 = list_D[0]
 
-    if styles is None and n_dict > 1:
+    if styles is None and n_dict >= 1:
         styles = [dict(color=f'C{i}') for i in range(n_dict)]
 
     # compute layout
@@ -202,6 +202,7 @@ class RotationAwareAnnotation(mpl_text.Annotation):
         Key-word arguments for the Annotation. List of available kwargs:
         https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.annotate.html
     """
+
     def __init__(self, text, anchor_pt, next_pt, ax=None, **kwargs):
         # Get the Artiste to draw on
         self.ax = ax or plt.gca()

docs/Makefile

@@ -3,15 +3,31 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?=
+SPHINXOPTS    +=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
+HOSTFILE      = ../hostfile
+ALLOW_AS_ROOT = ${ALLOW} 
 
 # Put it first so that "make" without argument is like "make help".
+.PHONY: help
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+	rm -rf auto_examples/*
+	rm -rf generated/*
+
+
+.PHONY: html
+html:
+	mpiexec -np 1 $(ALLOW_AS_ROOT) --hostfile $(HOSTFILE) $(SPHINXBUILD) -b html $(SOURCEDIR) $(BUILDDIR)/html $(SPHINXOPTS) ${0}
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
 .PHONY: help Makefile
 
 # Catch-all target: route all unknown targets to Sphinx using the new

docs/api.rst

@@ -4,7 +4,7 @@
 API Documentation
 =================
 
-.. currentmodule:: dicodile.dicodile
+.. currentmodule:: dicodile
 
 
 .. autosummary::

examples/plot_mandrill.py

@@ -0,0 +1,100 @@
+"""DiCoDiLe on the Mandrill image
+==============================
+
+This example illlustrates reconstruction of `Mandrill image
+<http://sipi.usc.edu/database/download.php?vol=misc&img=4.2.03>`_
+using DiCoDiLe algorithm with default soft_lock value "border" and 9
+workers.
+
+"""  # noqa
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+from dicodile.data.images import fetch_mandrill
+
+from dicodile.utils.dictionary import init_dictionary
+from dicodile.utils.viz import display_dictionaries
+from dicodile.utils.csc import reconstruct
+
+from dicodile import dicodile
+
+
+###############################################################################
+# We will first download the Mandrill image.
+
+X = fetch_mandrill()
+
+plt.axis('off')
+plt.imshow(X.swapaxes(0, 2))
+
+
+###############################################################################
+# We will create a random dictionary of **K = 25** patches of size **8x8**
+# from the original Mandrill image to be used for sparse coding.
+
+# set dictionary size
+n_atoms = 25
+
+# set individual atom (patch) size
+atom_support = (8, 8)
+
+D_init = init_dictionary(X, n_atoms, atom_support, random_state=60)
+
+###############################################################################
+# We are going to run `dicodile` with **9** workers on **3x3** grids.
+
+# number of iterations for dicodile
+n_iter = 3
+
+# number of iterations for csc (dicodile_z)
+max_iter = 10000
+
+# number of splits along each dimension
+w_world = 3
+
+# number of workers
+n_workers = w_world * w_world
+
+# coordinate selection strategy for coordinate descent
+strategy = 'greedy'
+
+###############################################################################
+# Run `dicodile`.
+
+pobj, times, D_hat, z_hat = dicodile(X, D_init, n_iter=n_iter,
+                                     n_workers=n_workers,
+                                     strategy=strategy,
+                                     dicod_kwargs={"max_iter": max_iter},
+                                     verbose=6)
+
+
+print("[DICOD] final cost : {}".format(pobj))
+
+###############################################################################
+# Plot and compare the initial dictionary `D_init` with the
+# dictionary `D_hat` improved by `dicodile`.
+
+# normalize dictionaries
+normalized_D_init = D_init / D_init.max()
+normalized_D_hat = D_hat / D_hat.max()
+
+display_dictionaries(normalized_D_init, normalized_D_hat)
+
+
+###############################################################################
+# Reconstruct the image from `z_hat` and `D_hat`.
+
+X_hat = reconstruct(z_hat, D_hat)
+X_hat = np.clip(X_hat, 0, 1)
+
+
+###############################################################################
+# Plot the reconstructed image.
+
+fig = plt.figure("recovery")
+
+ax = plt.subplot()
+ax.imshow(X_hat.swapaxes(0, 2))
+ax.axis('off')
+plt.tight_layout()