Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -10,6 +10,7 @@ from transformers import GLPNForDepthEstimation, GLPNImageProcessor
|
|
| 10 |
|
| 11 |
import open3d as o3d
|
| 12 |
|
|
|
|
| 13 |
# ------------------------------
|
| 14 |
# Model setup (loaded once)
|
| 15 |
# ------------------------------
|
|
@@ -18,6 +19,7 @@ FE = GLPNImageProcessor.from_pretrained("vinvino02/glpn-nyu")
|
|
| 18 |
MODEL = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-nyu").to(DEVICE)
|
| 19 |
MODEL.eval()
|
| 20 |
|
|
|
|
| 21 |
# ------------------------------
|
| 22 |
# Utilities
|
| 23 |
# ------------------------------
|
|
@@ -30,6 +32,7 @@ def _resize_to_mult32(img: Image.Image, max_h=480):
|
|
| 30 |
new_w = new_w - diff if diff < 16 else new_w + (32 - diff)
|
| 31 |
return img.resize((new_w, new_h), Image.BICUBIC)
|
| 32 |
|
|
|
|
| 33 |
def predict_depth(image_pil: Image.Image):
|
| 34 |
"""Run GLPN and return cropped RGB (as PIL) + raw depth (float32 numpy)."""
|
| 35 |
img = _resize_to_mult32(image_pil.convert("RGB"))
|
|
@@ -47,11 +50,13 @@ def predict_depth(image_pil: Image.Image):
|
|
| 47 |
rgb = img.crop((pad, pad, img.width - pad, img.height - pad))
|
| 48 |
return rgb, depth
|
| 49 |
|
|
|
|
| 50 |
def depth_to_colormap(depth: np.ndarray):
|
| 51 |
"""Return a PIL image (plasma colormap) from depth for preview."""
|
| 52 |
import matplotlib
|
| 53 |
matplotlib.use("Agg")
|
| 54 |
import matplotlib.pyplot as plt
|
|
|
|
| 55 |
|
| 56 |
d = depth.copy()
|
| 57 |
d -= d.min()
|
|
@@ -59,12 +64,10 @@ def depth_to_colormap(depth: np.ndarray):
|
|
| 59 |
d /= d.max()
|
| 60 |
d8 = (d * 255).astype(np.uint8)
|
| 61 |
|
| 62 |
-
|
| 63 |
-
import matplotlib.cm as cm
|
| 64 |
-
cmap = cm.get_cmap("plasma")
|
| 65 |
-
colored = (cmap(d8)[:, :, :3] * 255).astype(np.uint8)
|
| 66 |
return Image.fromarray(colored)
|
| 67 |
|
|
|
|
| 68 |
def rgbd_to_pointcloud(rgb_pil: Image.Image, depth: np.ndarray):
|
| 69 |
"""Create an Open3D point cloud from RGB + relative depth."""
|
| 70 |
# Normalize depth to 0..1 then to 0..255 uint8 for Open3D RGBD convenience
|
|
@@ -98,6 +101,7 @@ def rgbd_to_pointcloud(rgb_pil: Image.Image, depth: np.ndarray):
|
|
| 98 |
pcd.orient_normals_to_align_with_direction()
|
| 99 |
return pcd
|
| 100 |
|
|
|
|
| 101 |
def pointcloud_to_mesh(pcd: o3d.geometry.PointCloud, depth=10):
|
| 102 |
if len(pcd.points) == 0:
|
| 103 |
return None
|
|
@@ -110,6 +114,7 @@ def pointcloud_to_mesh(pcd: o3d.geometry.PointCloud, depth=10):
|
|
| 110 |
mesh.compute_vertex_normals()
|
| 111 |
return mesh
|
| 112 |
|
|
|
|
| 113 |
def save_o3d(obj, path):
|
| 114 |
ext = os.path.splitext(path)[1].lower()
|
| 115 |
if isinstance(obj, o3d.geometry.PointCloud):
|
|
@@ -118,15 +123,14 @@ def save_o3d(obj, path):
|
|
| 118 |
else:
|
| 119 |
raise ValueError("Point cloud: please save as .ply")
|
| 120 |
elif isinstance(obj, o3d.geometry.TriangleMesh):
|
| 121 |
-
if ext
|
| 122 |
-
o3d.io.write_triangle_mesh(path, obj)
|
| 123 |
-
elif ext == ".ply":
|
| 124 |
o3d.io.write_triangle_mesh(path, obj)
|
| 125 |
else:
|
| 126 |
raise ValueError("Mesh: use .obj or .ply")
|
| 127 |
else:
|
| 128 |
raise ValueError("Unsupported type for saving")
|
| 129 |
|
|
|
|
| 130 |
def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
|
| 131 |
"""
|
| 132 |
Try offscreen render for a preview PNG. If it fails (e.g., no EGL/OSMesa),
|
|
@@ -135,7 +139,7 @@ def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
|
|
| 135 |
try:
|
| 136 |
from open3d.visualization import rendering
|
| 137 |
|
| 138 |
-
#
|
| 139 |
if not mesh.has_vertex_colors():
|
| 140 |
mesh.paint_uniform_color([0.8, 0.8, 0.85])
|
| 141 |
|
|
@@ -152,7 +156,6 @@ def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
|
|
| 152 |
extent = bbox.get_extent()
|
| 153 |
radius = np.linalg.norm(extent) * 0.8 + 1e-6
|
| 154 |
|
| 155 |
-
# Camera looking at center from +z
|
| 156 |
cam = scene.camera
|
| 157 |
cam.look_at(center, center + [0, 0, radius], [0, 1, 0])
|
| 158 |
|
|
@@ -162,6 +165,7 @@ def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
|
|
| 162 |
except Exception:
|
| 163 |
return None
|
| 164 |
|
|
|
|
| 165 |
# ------------------------------
|
| 166 |
# Gradio pipeline
|
| 167 |
# ------------------------------
|
|
@@ -206,14 +210,15 @@ def run_pipeline(image: Image.Image, poisson_depth: int = 10):
|
|
| 206 |
|
| 207 |
return depth_vis, preview, pcd_path, mesh_obj_path
|
| 208 |
|
|
|
|
| 209 |
# ------------------------------
|
| 210 |
# Interface
|
| 211 |
# ------------------------------
|
| 212 |
TITLE = "Monocular Depth β Point Cloud β Poisson Mesh (GLPN + Open3D)"
|
| 213 |
DESC = """
|
| 214 |
Upload an image. We estimate relative depth (GLPN), build a point cloud, and reconstruct
|
| 215 |
-
a mesh (Poisson). Outputs: depth preview, mesh preview (if renderer available),
|
| 216 |
-
and downloads for .ply (point cloud) and .obj (mesh).
|
| 217 |
**Note:** monocular depth lacks absolute scale; this is for visualization/demo purposes.
|
| 218 |
"""
|
| 219 |
|
|
@@ -223,7 +228,12 @@ with gr.Blocks(title="2D β 3D Reconstruction") as demo:
|
|
| 223 |
|
| 224 |
with gr.Row():
|
| 225 |
with gr.Column():
|
| 226 |
-
in_img = gr.Image(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
poisson_depth = gr.Slider(5, 12, value=10, step=1, label="Poisson depth (mesh detail)")
|
| 228 |
run_btn = gr.Button("Reconstruct 3D", variant="primary")
|
| 229 |
|
|
@@ -245,4 +255,5 @@ with gr.Blocks(title="2D β 3D Reconstruction") as demo:
|
|
| 245 |
mesh_obj_view.change(lambda p: p, inputs=mesh_obj_view, outputs=mesh_obj_file)
|
| 246 |
|
| 247 |
if __name__ == "__main__":
|
| 248 |
-
|
|
|
|
|
|
| 10 |
|
| 11 |
import open3d as o3d
|
| 12 |
|
| 13 |
+
|
| 14 |
# ------------------------------
|
| 15 |
# Model setup (loaded once)
|
| 16 |
# ------------------------------
|
|
|
|
| 19 |
MODEL = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-nyu").to(DEVICE)
|
| 20 |
MODEL.eval()
|
| 21 |
|
| 22 |
+
|
| 23 |
# ------------------------------
|
| 24 |
# Utilities
|
| 25 |
# ------------------------------
|
|
|
|
| 32 |
new_w = new_w - diff if diff < 16 else new_w + (32 - diff)
|
| 33 |
return img.resize((new_w, new_h), Image.BICUBIC)
|
| 34 |
|
| 35 |
+
|
| 36 |
def predict_depth(image_pil: Image.Image):
|
| 37 |
"""Run GLPN and return cropped RGB (as PIL) + raw depth (float32 numpy)."""
|
| 38 |
img = _resize_to_mult32(image_pil.convert("RGB"))
|
|
|
|
| 50 |
rgb = img.crop((pad, pad, img.width - pad, img.height - pad))
|
| 51 |
return rgb, depth
|
| 52 |
|
| 53 |
+
|
| 54 |
def depth_to_colormap(depth: np.ndarray):
|
| 55 |
"""Return a PIL image (plasma colormap) from depth for preview."""
|
| 56 |
import matplotlib
|
| 57 |
matplotlib.use("Agg")
|
| 58 |
import matplotlib.pyplot as plt
|
| 59 |
+
import matplotlib.cm as cm
|
| 60 |
|
| 61 |
d = depth.copy()
|
| 62 |
d -= d.min()
|
|
|
|
| 64 |
d /= d.max()
|
| 65 |
d8 = (d * 255).astype(np.uint8)
|
| 66 |
|
| 67 |
+
colored = (cm.get_cmap("plasma")(d8)[:, :, :3] * 255).astype(np.uint8)
|
|
|
|
|
|
|
|
|
|
| 68 |
return Image.fromarray(colored)
|
| 69 |
|
| 70 |
+
|
| 71 |
def rgbd_to_pointcloud(rgb_pil: Image.Image, depth: np.ndarray):
|
| 72 |
"""Create an Open3D point cloud from RGB + relative depth."""
|
| 73 |
# Normalize depth to 0..1 then to 0..255 uint8 for Open3D RGBD convenience
|
|
|
|
| 101 |
pcd.orient_normals_to_align_with_direction()
|
| 102 |
return pcd
|
| 103 |
|
| 104 |
+
|
| 105 |
def pointcloud_to_mesh(pcd: o3d.geometry.PointCloud, depth=10):
|
| 106 |
if len(pcd.points) == 0:
|
| 107 |
return None
|
|
|
|
| 114 |
mesh.compute_vertex_normals()
|
| 115 |
return mesh
|
| 116 |
|
| 117 |
+
|
| 118 |
def save_o3d(obj, path):
|
| 119 |
ext = os.path.splitext(path)[1].lower()
|
| 120 |
if isinstance(obj, o3d.geometry.PointCloud):
|
|
|
|
| 123 |
else:
|
| 124 |
raise ValueError("Point cloud: please save as .ply")
|
| 125 |
elif isinstance(obj, o3d.geometry.TriangleMesh):
|
| 126 |
+
if ext in {".obj", ".ply"}:
|
|
|
|
|
|
|
| 127 |
o3d.io.write_triangle_mesh(path, obj)
|
| 128 |
else:
|
| 129 |
raise ValueError("Mesh: use .obj or .ply")
|
| 130 |
else:
|
| 131 |
raise ValueError("Unsupported type for saving")
|
| 132 |
|
| 133 |
+
|
| 134 |
def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
|
| 135 |
"""
|
| 136 |
Try offscreen render for a preview PNG. If it fails (e.g., no EGL/OSMesa),
|
|
|
|
| 139 |
try:
|
| 140 |
from open3d.visualization import rendering
|
| 141 |
|
| 142 |
+
# Ensure it has some color
|
| 143 |
if not mesh.has_vertex_colors():
|
| 144 |
mesh.paint_uniform_color([0.8, 0.8, 0.85])
|
| 145 |
|
|
|
|
| 156 |
extent = bbox.get_extent()
|
| 157 |
radius = np.linalg.norm(extent) * 0.8 + 1e-6
|
| 158 |
|
|
|
|
| 159 |
cam = scene.camera
|
| 160 |
cam.look_at(center, center + [0, 0, radius], [0, 1, 0])
|
| 161 |
|
|
|
|
| 165 |
except Exception:
|
| 166 |
return None
|
| 167 |
|
| 168 |
+
|
| 169 |
# ------------------------------
|
| 170 |
# Gradio pipeline
|
| 171 |
# ------------------------------
|
|
|
|
| 210 |
|
| 211 |
return depth_vis, preview, pcd_path, mesh_obj_path
|
| 212 |
|
| 213 |
+
|
| 214 |
# ------------------------------
|
| 215 |
# Interface
|
| 216 |
# ------------------------------
|
| 217 |
TITLE = "Monocular Depth β Point Cloud β Poisson Mesh (GLPN + Open3D)"
|
| 218 |
DESC = """
|
| 219 |
Upload an image. We estimate relative depth (GLPN), build a point cloud, and reconstruct
|
| 220 |
+
a mesh (Poisson). Outputs: depth preview, mesh preview (if renderer available),
|
| 221 |
+
and downloads for .ply (point cloud) and .obj (mesh).
|
| 222 |
**Note:** monocular depth lacks absolute scale; this is for visualization/demo purposes.
|
| 223 |
"""
|
| 224 |
|
|
|
|
| 228 |
|
| 229 |
with gr.Row():
|
| 230 |
with gr.Column():
|
| 231 |
+
in_img = gr.Image(
|
| 232 |
+
type="pil",
|
| 233 |
+
sources=["upload", "clipboard"],
|
| 234 |
+
label="Input Image",
|
| 235 |
+
image_mode="RGB"
|
| 236 |
+
)
|
| 237 |
poisson_depth = gr.Slider(5, 12, value=10, step=1, label="Poisson depth (mesh detail)")
|
| 238 |
run_btn = gr.Button("Reconstruct 3D", variant="primary")
|
| 239 |
|
|
|
|
| 255 |
mesh_obj_view.change(lambda p: p, inputs=mesh_obj_view, outputs=mesh_obj_file)
|
| 256 |
|
| 257 |
if __name__ == "__main__":
|
| 258 |
+
# share=True creates a public link (useful on Spaces/Colab/local)
|
| 259 |
+
demo.launch(share=True)
|