Project4 -- Image-based Lighting

eric11220 · Jan 14, 2020 · c27c87d · c27c87d
1 parent 0b2b12c
commit c27c87d
Show file tree

Hide file tree

Showing 84 changed files with 3,245 additions and 0 deletions.
diff --git a/proj4-image_based_lighting/Project 4. Image-Based Lighting - Template.ipynb b/proj4-image_based_lighting/Project 4. Image-Based Lighting - Template.ipynb
diff --git a/proj4-image_based_lighting/images/.DS_Store b/proj4-image_based_lighting/images/.DS_Store
diff --git a/proj4-image_based_lighting/images/inputs/.DS_Store b/proj4-image_based_lighting/images/inputs/.DS_Store
diff --git a/proj4-image_based_lighting/images/outputs/.DS_Store b/proj4-image_based_lighting/images/outputs/.DS_Store
diff --git a/proj4-image_based_lighting/images/outputs/cubic.png b/proj4-image_based_lighting/images/outputs/cubic.png
diff --git a/proj4-image_based_lighting/images/outputs/equirectangular.hdr b/proj4-image_based_lighting/images/outputs/equirectangular.hdr
diff --git a/proj4-image_based_lighting/images/outputs/equirectangular.png b/proj4-image_based_lighting/images/outputs/equirectangular.png
diff --git a/proj4-image_based_lighting/images/outputs/estimated_functions.png b/proj4-image_based_lighting/images/outputs/estimated_functions.png
diff --git a/proj4-image_based_lighting/images/outputs/estimated_hdr.hdr b/proj4-image_based_lighting/images/outputs/estimated_hdr.hdr
diff --git a/proj4-image_based_lighting/images/outputs/estimation_0.png b/proj4-image_based_lighting/images/outputs/estimation_0.png
diff --git a/proj4-image_based_lighting/images/outputs/estimation_1.png b/proj4-image_based_lighting/images/outputs/estimation_1.png
diff --git a/proj4-image_based_lighting/images/outputs/estimation_2.png b/proj4-image_based_lighting/images/outputs/estimation_2.png
diff --git a/proj4-image_based_lighting/images/outputs/estimation_hdr.png b/proj4-image_based_lighting/images/outputs/estimation_hdr.png
diff --git a/proj4-image_based_lighting/images/outputs/filtered_0.png b/proj4-image_based_lighting/images/outputs/filtered_0.png
diff --git a/proj4-image_based_lighting/images/outputs/filtered_1.png b/proj4-image_based_lighting/images/outputs/filtered_1.png
diff --git a/proj4-image_based_lighting/images/outputs/filtered_2.png b/proj4-image_based_lighting/images/outputs/filtered_2.png
diff --git a/proj4-image_based_lighting/images/outputs/filtered_hdr.hdr b/proj4-image_based_lighting/images/outputs/filtered_hdr.hdr
diff --git a/proj4-image_based_lighting/images/outputs/filtered_hdr.png b/proj4-image_based_lighting/images/outputs/filtered_hdr.png
diff --git a/proj4-image_based_lighting/images/outputs/inpainted.png b/proj4-image_based_lighting/images/outputs/inpainted.png
diff --git a/proj4-image_based_lighting/images/outputs/merged.png b/proj4-image_based_lighting/images/outputs/merged.png
diff --git a/proj4-image_based_lighting/images/outputs/naive_0.png b/proj4-image_based_lighting/images/outputs/naive_0.png
diff --git a/proj4-image_based_lighting/images/outputs/naive_1.png b/proj4-image_based_lighting/images/outputs/naive_1.png
diff --git a/proj4-image_based_lighting/images/outputs/naive_2.png b/proj4-image_based_lighting/images/outputs/naive_2.png
diff --git a/proj4-image_based_lighting/images/outputs/naive_hdr.hdr b/proj4-image_based_lighting/images/outputs/naive_hdr.hdr
diff --git a/proj4-image_based_lighting/images/outputs/naive_hdr.png b/proj4-image_based_lighting/images/outputs/naive_hdr.png
diff --git a/proj4-image_based_lighting/images/project_4/.DS_Store b/proj4-image_based_lighting/images/project_4/.DS_Store
diff --git a/proj4-image_based_lighting/images/project_4/10_add_equirectangular_lighting.gif b/proj4-image_based_lighting/images/project_4/10_add_equirectangular_lighting.gif
diff --git a/proj4-image_based_lighting/images/project_4/11_render_cycles.gif b/proj4-image_based_lighting/images/project_4/11_render_cycles.gif
diff --git a/proj4-image_based_lighting/images/project_4/12_save_mask.gif b/proj4-image_based_lighting/images/project_4/12_save_mask.gif
diff --git a/proj4-image_based_lighting/images/project_4/1_delete_initial.gif b/proj4-image_based_lighting/images/project_4/1_delete_initial.gif
diff --git a/proj4-image_based_lighting/images/project_4/2_set_camera_position.gif b/proj4-image_based_lighting/images/project_4/2_set_camera_position.gif
diff --git a/proj4-image_based_lighting/images/project_4/3_go_into_camera.gif b/proj4-image_based_lighting/images/project_4/3_go_into_camera.gif
diff --git a/proj4-image_based_lighting/images/project_4/4_setup_dimensions.gif b/proj4-image_based_lighting/images/project_4/4_setup_dimensions.gif
diff --git a/proj4-image_based_lighting/images/project_4/5_load_background_and_align.gif b/proj4-image_based_lighting/images/project_4/5_load_background_and_align.gif
diff --git a/proj4-image_based_lighting/images/project_4/6_adding_plane.gif b/proj4-image_based_lighting/images/project_4/6_adding_plane.gif
diff --git a/proj4-image_based_lighting/images/project_4/7_setting_plane_material.gif b/proj4-image_based_lighting/images/project_4/7_setting_plane_material.gif
diff --git a/proj4-image_based_lighting/images/project_4/8_adding_mesh.gif b/proj4-image_based_lighting/images/project_4/8_adding_mesh.gif
diff --git a/proj4-image_based_lighting/images/project_4/9_set_mesh_material.gif b/proj4-image_based_lighting/images/project_4/9_set_mesh_material.gif
diff --git a/proj4-image_based_lighting/images/project_4/background.jpg b/proj4-image_based_lighting/images/project_4/background.jpg
diff --git a/proj4-image_based_lighting/images/project_4/background_with_ball.jpg b/proj4-image_based_lighting/images/project_4/background_with_ball.jpg
diff --git a/proj4-image_based_lighting/images/project_4/cropped.jpg b/proj4-image_based_lighting/images/project_4/cropped.jpg
diff --git a/proj4-image_based_lighting/images/project_4/ldr2hdr.png b/proj4-image_based_lighting/images/project_4/ldr2hdr.png
diff --git a/proj4-image_based_lighting/images/project_4/local_tonemap.jpg b/proj4-image_based_lighting/images/project_4/local_tonemap.jpg
diff --git a/proj4-image_based_lighting/images/project_4/panoramic_trans.jpg b/proj4-image_based_lighting/images/project_4/panoramic_trans.jpg
diff --git a/proj4-image_based_lighting/images/project_4/render_composite.jpg b/proj4-image_based_lighting/images/project_4/render_composite.jpg
diff --git a/proj4-image_based_lighting/images/project_4/render_empty.jpg b/proj4-image_based_lighting/images/project_4/render_empty.jpg
diff --git a/proj4-image_based_lighting/images/project_4/render_mask.jpg b/proj4-image_based_lighting/images/project_4/render_mask.jpg
diff --git a/proj4-image_based_lighting/images/project_4/render_render.jpg b/proj4-image_based_lighting/images/project_4/render_render.jpg
diff --git a/proj4-image_based_lighting/index.html b/proj4-image_based_lighting/index.html
@@ -0,0 +1,111 @@
+<!doctype html>
+<head>
+ 	<link rel="stylesheet" href="styles.css">
+</head>
+
+<body>
+<h1>Image-Based Lighting</h1>
+<h3>HDR Images</h3>
+No matter at which stage, the pixel value at (128, 128) of the three log irradiance images are different. For the naive stage, they are [190, 189, 188], [205, 205, 203] and [221, 219, 216]; for the filtered stage, they are [205, 227, 224], [216, 216, 214], [228, 227, 224]; for the estimation stage, they are [104, 96, 88], [93, 98, 93] and [114, 110, 104]. The reason is the exposure time for the three images are different and since ln(Z) = ln(R) + ln(t), the pixel values are different. And for the estimated stage, even though g(Z) = ln(R) + ln(t) and that g(.) is a non-linear function, given that ln(R) + ln(t) is different from image to image, Z is highly likely to be different.
+
+<br><br>
+<table style="text-align:center; border-spacing: 15px 5px">
+	<tr>
+		<td> LDR Image (1/5) </td>
+		<td> LDR Image (1/40) </td>
+		<td> LDR Image (1/160)</td>
+		<td> Background Image</td>
+	</tr>
+	<tr>
+		<td><img src="samples/1_5_cut.png" height="192" width="192"></img></td>
+		<td><img src="samples/1_40_cut.png" height="192" width="192"></img></td>
+		<td><img src="samples/1_160_cut.png" height="192" width="192"></img></td>
+		<td><img src="samples/my_empty.jpg" height="144" width="192" style="transform:rotate(90deg);"></img></td>
+	</tr>
+	<tr>
+		<td colspan="3">Naive Log Irradiances</td>
+		<td>HDR Image</td>
+	</tr>
+    <tr>
+		<td><img src="images/outputs/naive_0.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/naive_1.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/naive_2.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/naive_hdr.png" height="192" width="192"></img></td>
+	</tr>
+	<tr>
+		<td colspan="3">Filtered Log Irradiances</td>
+		<td>HDR Image</td>
+	</tr>
+    <tr>
+		<td><img src="images/outputs/filtered_0.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/filtered_1.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/filtered_2.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/filtered_hdr.png" height="192" width="192"></img></td>
+	</tr>
+	<tr>
+		<td colspan="3">Estimated Log Irradiances</td>
+		<td>HDR Image</td>
+	</tr>
+    <tr>
+		<td><img src="images/outputs/estimation_0.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/estimation_1.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/estimation_2.png" height="192" width="192"></img></td>
+		<td><img src="images/outputs/estimation_hdr.png" height="192" width="192"></img></td>
+	</tr>
+</table>
+
+<h3>Estimated Function g</h3>
+<img src="images/outputs/estimated_functions.png">
+
+<h3>Equirectangular HDR Image</h3>
+We follow the post in Piazza, setting x-axis to right, y-axis up and z-axis away from the image. Firstly, we compute the normal by calculating the x and y component first using the pixel distance from the center of the image (which we assume to be the center of the ball) and z component with formula <i>z^2 = sqrt(1 - y^2 - x^2)</i>. With the normal, we could derive the reflection vector using the formula provided in the slides by implementing a function for dot product between two 3-d vectors. Then with the reflection vector, we compute phi and theta using <i>arctan(z/x) + pi</i> and <i>arccos(y)</i> respectively. Finally, with phi and theta, we follow the instructions to interpolate the equirectangular image using <i>griddata</i>.
+<br><br>
+<img src="images/outputs/equirectangular.png" height="270" width="540">
+
+<h3>Composite Result</h3>
+We show two sets fo compositing images. Both are with the same background but the second one is rendered with a new object -- sofa.
+
+<br><br>
+<table style="text-align:center; border-spacing: 15px 5px">
+	<tr>
+		<td> Background Image </td>
+		<td> Compositing Result </td>
+		<td> Render w/ Objects </td>
+		<td> Render w/o Objects </td>
+		<td> Object Mask </td>
+	</tr>
+	<tr>
+		<td><img src="samples/my_empty.jpg" height="192" width="256" style="transform:rotate(90deg);"></img></td>
+		<td><img src="images/outputs/merged.png" height="256" width="192"></img></td>
+		<td><img src="samples/r1.png" height="256" width="192"></img></td>
+		<td><img src="samples/r2.png" height="256" width="192"></img></td>
+		<td><img src="samples/mask.png" height="256" width="192"></img></td>
+	</tr>
+	<tr>
+		<td></td>
+		<td><img src="images/outputs/merged_2.png" height="256" width="192"></img></td>
+		<td><img src="samples/r4.png" height="256" width="192"></img></td>
+		<td><img src="samples/r3.png" height="256" width="192"></img></td>
+		<td><img src="samples/mask_2.png" height="256" width="192"></img></td>
+	</tr>
+</table>
+
+<h1>Bells and Whistles</h1>
+<h3>Other Panoramic transformation (Cubic)</h3>
+I implement the pseudo codes provided on the Wiki page and yet has to flip each image upside down.
+<br><br>
+<img src="images/outputs/cubic.png" height="384" width="512">
+
+<h3>Photographer removal</h3>
+I implement the first method, that is, using the image inpainting method we've described and implemented in proj. 2 to remove and fill in the masked area.
+<br><br>
+<table style="text-align:center; border-spacing: 15px 5px">
+	<tr>
+		<td> Original </td>
+		<td> Inpainted </td>
+	</tr>
+	<tr>
+		<td><img src="images/outputs/equirectangular.png" height="270" width="540"></td>
+		<td><img src="images/outputs/inpainted.png" height="270" width="540"></td>
+	</tr>
+</body>
diff --git a/proj4-image_based_lighting/proj4.pdf b/proj4-image_based_lighting/proj4.pdf
diff --git a/proj4-image_based_lighting/samples/.DS_Store b/proj4-image_based_lighting/samples/.DS_Store
diff --git a/proj4-image_based_lighting/samples/0024.jpg b/proj4-image_based_lighting/samples/0024.jpg
diff --git a/proj4-image_based_lighting/samples/0060.jpg b/proj4-image_based_lighting/samples/0060.jpg
diff --git a/proj4-image_based_lighting/samples/0120.jpg b/proj4-image_based_lighting/samples/0120.jpg
diff --git a/proj4-image_based_lighting/samples/0205.jpg b/proj4-image_based_lighting/samples/0205.jpg
diff --git a/proj4-image_based_lighting/samples/0553.jpg b/proj4-image_based_lighting/samples/0553.jpg
diff --git a/proj4-image_based_lighting/samples/1_160_cut.png b/proj4-image_based_lighting/samples/1_160_cut.png
diff --git a/proj4-image_based_lighting/samples/1_40_cut.png b/proj4-image_based_lighting/samples/1_40_cut.png
diff --git a/proj4-image_based_lighting/samples/1_5_cut.png b/proj4-image_based_lighting/samples/1_5_cut.png
diff --git a/proj4-image_based_lighting/samples/empty.jpg b/proj4-image_based_lighting/samples/empty.jpg
diff --git a/proj4-image_based_lighting/samples/ibl.blend b/proj4-image_based_lighting/samples/ibl.blend
diff --git a/proj4-image_based_lighting/samples/ibl_script.py b/proj4-image_based_lighting/samples/ibl_script.py
@@ -0,0 +1,130 @@
+# CS 445 Computational Photography  https://courses.engr.illinois.edu/cs445/
+# Project 4: Image-based Lighting
+
+import bpy
+from os import path
+
+
+def is_plane(obj):
+    """
+    :return: True if the given object's mesh has exactly 4 vertices and edges.
+    """
+    mesh = obj.data
+    return hasattr(mesh, 'edges') and hasattr(mesh, 'vertices') and len(
+        mesh.edges) == 4 and len(mesh.vertices) == 4
+
+
+def foreground_objects():
+    """
+    :return: List of all renderable objects excluding any plane(s).
+    """
+    foreground = []
+    for obj in bpy.data.objects:
+        if (not hasattr(obj.data, 'materials') or is_plane(obj) or
+                issubclass(type(obj.data), bpy.types.Lamp)):
+            continue
+        foreground.append(obj)
+    return foreground
+
+
+def render_and_save(png_filename, out_dir='./'):
+    """
+    Renders the scene (using the currently selected rendering engine)
+    and saves the image.
+
+    :param png_path: Path to output image
+    """
+    from os import path
+    out_path = path.join(path.expanduser(out_dir), png_filename)
+    bpy.ops.render.render()
+    bpy.data.images['Render Result'].save_render(filepath=out_path)
+
+
+def object_mask_mode():
+    """
+    Removes all existing materials and assigns a mask.
+    """
+
+    def create_mask(name='MaskMaterial', rgb=(1.0, 1.0, 1.0)):
+        mask = bpy.data.materials.new(name)
+        mask.specular_color = rgb
+        mask.specular_intensity = 1.0
+        mask.use_shadeless = True
+        return mask
+
+    # Uses Blender Renderer
+    bpy.context.scene.render.engine = 'BLENDER_RENDER'
+    bpy.ops.object.mode_set(mode='OBJECT')
+
+    # Create a mask material.
+    mask = create_mask()
+    for obj in foreground_objects():
+        obj.data.materials.clear()
+        obj.data.materials.append(mask)
+        print('Assigning mask material to {}'.format(obj.data.name))
+        obj.hide_render = False
+
+    # Sets Horizon RGB color to black.
+    bpy.context.scene.world.horizon_color = (0, 0, 0)
+
+
+def object_rendering_mode(environment_texture_path,
+                          hide_objects=False,
+                          background_strength=0.01,
+                          surface_rgba=(0.367, 0.310, 0.298, 1.0)):
+    """
+    Prepares the scene for rendering with Cycles renderer.
+
+    :param hide_objects: If true, only the plane surface will be visible.
+    :param environment_texture_path: Path to environment texture image.
+    :param background_strength: Environment light intensity.
+    :param surface_rgba: Color of the plane.
+    """
+    bpy.context.scene.render.engine = 'CYCLES'
+    bpy.ops.object.mode_set(mode='OBJECT')
+
+    for obj in foreground_objects():
+        obj.hide_render = hide_objects
+
+    for obj in bpy.data.objects:
+        if is_plane(obj) and obj.active_material is not None:
+            node = obj.active_material.node_tree.nodes['Diffuse BSDF']
+            node.inputs['Color'].default_value = surface_rgba
+
+    world_background = bpy.context.scene.world.node_tree.nodes['Background']
+    world_background.inputs['Strength'].default_value = background_strength
+
+    image = bpy.data.images.load(environment_texture_path)
+    environment_texture = world_background.inputs['Color'].links[0].from_node
+    environment_texture.image = image
+
+## Blender script starts here.
+
+# Change this path:
+project_path = '/path/to/proj4_materials/'
+
+# Path to the HDR image.
+environment_texture = path.join(project_path, 'latlon.hdr')
+
+# Saves before rendering.
+bpy.ops.wm.save_mainfile()
+
+try:
+    print('Rendering objects.')
+    object_rendering_mode(environment_texture_path=environment_texture)
+    render_and_save('objects.png', out_dir=project_path)
+
+    print('Rendering without objects.')
+    object_rendering_mode(hide_objects=True,
+                          environment_texture_path=environment_texture)
+    render_and_save('without_objects.png', out_dir=project_path)
+
+    print('Rendering object mask.')
+    object_mask_mode()  # This step will remove all foreground object materials.
+    render_and_save('mask.png', out_dir=project_path)
+
+    print('Done')
+
+finally:
+    # Reverts back to the last saved state.
+    bpy.ops.wm.revert_mainfile()
diff --git a/proj4-image_based_lighting/samples/latlon.hdr b/proj4-image_based_lighting/samples/latlon.hdr
diff --git a/proj4-image_based_lighting/samples/mask.png b/proj4-image_based_lighting/samples/mask.png
diff --git a/proj4-image_based_lighting/samples/mask_2.png b/proj4-image_based_lighting/samples/mask_2.png
diff --git a/proj4-image_based_lighting/samples/merged_2.png b/proj4-image_based_lighting/samples/merged_2.png
diff --git a/proj4-image_based_lighting/samples/my_empty.jpg b/proj4-image_based_lighting/samples/my_empty.jpg
diff --git a/proj4-image_based_lighting/samples/r1.png b/proj4-image_based_lighting/samples/r1.png
diff --git a/proj4-image_based_lighting/samples/r2.png b/proj4-image_based_lighting/samples/r2.png
diff --git a/proj4-image_based_lighting/samples/r3.png b/proj4-image_based_lighting/samples/r3.png
diff --git a/proj4-image_based_lighting/samples/r4.png b/proj4-image_based_lighting/samples/r4.png
diff --git a/proj4-image_based_lighting/samples/readme.txt b/proj4-image_based_lighting/samples/readme.txt
@@ -0,0 +1,8 @@
+Directory contents:
+
+empty.jpg -            background photo used for inserting objects
+{0024,...,0553}.jpg -  mirrorball photos used to compute latlon.exr 
+                       (exposure times are 1/24s, ..., 1/553s)
+latlon.exr -           equirectangular HDR map used for relighting
+ibl.blend -            example blend file
+
diff --git a/proj4-image_based_lighting/utils/__init__.py b/proj4-image_based_lighting/utils/__init__.py
diff --git a/proj4-image_based_lighting/utils/__pycache__/__init__.cpython-36.pyc b/proj4-image_based_lighting/utils/__pycache__/__init__.cpython-36.pyc
diff --git a/proj4-image_based_lighting/utils/__pycache__/bilateral_filter.cpython-36.pyc b/proj4-image_based_lighting/utils/__pycache__/bilateral_filter.cpython-36.pyc
diff --git a/proj4-image_based_lighting/utils/__pycache__/display.cpython-36.pyc b/proj4-image_based_lighting/utils/__pycache__/display.cpython-36.pyc
diff --git a/proj4-image_based_lighting/utils/__pycache__/hdr_helpers.cpython-36.pyc b/proj4-image_based_lighting/utils/__pycache__/hdr_helpers.cpython-36.pyc
diff --git a/proj4-image_based_lighting/utils/__pycache__/io.cpython-36.pyc b/proj4-image_based_lighting/utils/__pycache__/io.cpython-36.pyc
diff --git a/proj4-image_based_lighting/utils/__pycache__/meta.cpython-36.pyc b/proj4-image_based_lighting/utils/__pycache__/meta.cpython-36.pyc