Tohru127 commited on
Commit
dc673c1
Β·
verified Β·
1 Parent(s): d65081d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +24 -13
app.py CHANGED
@@ -10,6 +10,7 @@ from transformers import GLPNForDepthEstimation, GLPNImageProcessor
10
 
11
  import open3d as o3d
12
 
 
13
  # ------------------------------
14
  # Model setup (loaded once)
15
  # ------------------------------
@@ -18,6 +19,7 @@ FE = GLPNImageProcessor.from_pretrained("vinvino02/glpn-nyu")
18
  MODEL = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-nyu").to(DEVICE)
19
  MODEL.eval()
20
 
 
21
  # ------------------------------
22
  # Utilities
23
  # ------------------------------
@@ -30,6 +32,7 @@ def _resize_to_mult32(img: Image.Image, max_h=480):
30
  new_w = new_w - diff if diff < 16 else new_w + (32 - diff)
31
  return img.resize((new_w, new_h), Image.BICUBIC)
32
 
 
33
  def predict_depth(image_pil: Image.Image):
34
  """Run GLPN and return cropped RGB (as PIL) + raw depth (float32 numpy)."""
35
  img = _resize_to_mult32(image_pil.convert("RGB"))
@@ -47,11 +50,13 @@ def predict_depth(image_pil: Image.Image):
47
  rgb = img.crop((pad, pad, img.width - pad, img.height - pad))
48
  return rgb, depth
49
 
 
50
  def depth_to_colormap(depth: np.ndarray):
51
  """Return a PIL image (plasma colormap) from depth for preview."""
52
  import matplotlib
53
  matplotlib.use("Agg")
54
  import matplotlib.pyplot as plt
 
55
 
56
  d = depth.copy()
57
  d -= d.min()
@@ -59,12 +64,10 @@ def depth_to_colormap(depth: np.ndarray):
59
  d /= d.max()
60
  d8 = (d * 255).astype(np.uint8)
61
 
62
- # Make a small PNG buffer
63
- import matplotlib.cm as cm
64
- cmap = cm.get_cmap("plasma")
65
- colored = (cmap(d8)[:, :, :3] * 255).astype(np.uint8)
66
  return Image.fromarray(colored)
67
 
 
68
  def rgbd_to_pointcloud(rgb_pil: Image.Image, depth: np.ndarray):
69
  """Create an Open3D point cloud from RGB + relative depth."""
70
  # Normalize depth to 0..1 then to 0..255 uint8 for Open3D RGBD convenience
@@ -98,6 +101,7 @@ def rgbd_to_pointcloud(rgb_pil: Image.Image, depth: np.ndarray):
98
  pcd.orient_normals_to_align_with_direction()
99
  return pcd
100
 
 
101
  def pointcloud_to_mesh(pcd: o3d.geometry.PointCloud, depth=10):
102
  if len(pcd.points) == 0:
103
  return None
@@ -110,6 +114,7 @@ def pointcloud_to_mesh(pcd: o3d.geometry.PointCloud, depth=10):
110
  mesh.compute_vertex_normals()
111
  return mesh
112
 
 
113
  def save_o3d(obj, path):
114
  ext = os.path.splitext(path)[1].lower()
115
  if isinstance(obj, o3d.geometry.PointCloud):
@@ -118,15 +123,14 @@ def save_o3d(obj, path):
118
  else:
119
  raise ValueError("Point cloud: please save as .ply")
120
  elif isinstance(obj, o3d.geometry.TriangleMesh):
121
- if ext == ".obj":
122
- o3d.io.write_triangle_mesh(path, obj)
123
- elif ext == ".ply":
124
  o3d.io.write_triangle_mesh(path, obj)
125
  else:
126
  raise ValueError("Mesh: use .obj or .ply")
127
  else:
128
  raise ValueError("Unsupported type for saving")
129
 
 
130
  def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
131
  """
132
  Try offscreen render for a preview PNG. If it fails (e.g., no EGL/OSMesa),
@@ -135,7 +139,7 @@ def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
135
  try:
136
  from open3d.visualization import rendering
137
 
138
- # Make sure mesh has vertex colors or a default material
139
  if not mesh.has_vertex_colors():
140
  mesh.paint_uniform_color([0.8, 0.8, 0.85])
141
 
@@ -152,7 +156,6 @@ def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
152
  extent = bbox.get_extent()
153
  radius = np.linalg.norm(extent) * 0.8 + 1e-6
154
 
155
- # Camera looking at center from +z
156
  cam = scene.camera
157
  cam.look_at(center, center + [0, 0, radius], [0, 1, 0])
158
 
@@ -162,6 +165,7 @@ def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
162
  except Exception:
163
  return None
164
 
 
165
  # ------------------------------
166
  # Gradio pipeline
167
  # ------------------------------
@@ -206,14 +210,15 @@ def run_pipeline(image: Image.Image, poisson_depth: int = 10):
206
 
207
  return depth_vis, preview, pcd_path, mesh_obj_path
208
 
 
209
  # ------------------------------
210
  # Interface
211
  # ------------------------------
212
  TITLE = "Monocular Depth β†’ Point Cloud β†’ Poisson Mesh (GLPN + Open3D)"
213
  DESC = """
214
  Upload an image. We estimate relative depth (GLPN), build a point cloud, and reconstruct
215
- a mesh (Poisson). Outputs: depth preview, mesh preview (if renderer available),
216
- and downloads for .ply (point cloud) and .obj (mesh).
217
  **Note:** monocular depth lacks absolute scale; this is for visualization/demo purposes.
218
  """
219
 
@@ -223,7 +228,12 @@ with gr.Blocks(title="2D β†’ 3D Reconstruction") as demo:
223
 
224
  with gr.Row():
225
  with gr.Column():
226
- in_img = gr.Image(type="pil", label="Input Image")
 
 
 
 
 
227
  poisson_depth = gr.Slider(5, 12, value=10, step=1, label="Poisson depth (mesh detail)")
228
  run_btn = gr.Button("Reconstruct 3D", variant="primary")
229
 
@@ -245,4 +255,5 @@ with gr.Blocks(title="2D β†’ 3D Reconstruction") as demo:
245
  mesh_obj_view.change(lambda p: p, inputs=mesh_obj_view, outputs=mesh_obj_file)
246
 
247
  if __name__ == "__main__":
248
- demo.launch()
 
 
10
 
11
  import open3d as o3d
12
 
13
+
14
  # ------------------------------
15
  # Model setup (loaded once)
16
  # ------------------------------
 
19
  MODEL = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-nyu").to(DEVICE)
20
  MODEL.eval()
21
 
22
+
23
  # ------------------------------
24
  # Utilities
25
  # ------------------------------
 
32
  new_w = new_w - diff if diff < 16 else new_w + (32 - diff)
33
  return img.resize((new_w, new_h), Image.BICUBIC)
34
 
35
+
36
  def predict_depth(image_pil: Image.Image):
37
  """Run GLPN and return cropped RGB (as PIL) + raw depth (float32 numpy)."""
38
  img = _resize_to_mult32(image_pil.convert("RGB"))
 
50
  rgb = img.crop((pad, pad, img.width - pad, img.height - pad))
51
  return rgb, depth
52
 
53
+
54
  def depth_to_colormap(depth: np.ndarray):
55
  """Return a PIL image (plasma colormap) from depth for preview."""
56
  import matplotlib
57
  matplotlib.use("Agg")
58
  import matplotlib.pyplot as plt
59
+ import matplotlib.cm as cm
60
 
61
  d = depth.copy()
62
  d -= d.min()
 
64
  d /= d.max()
65
  d8 = (d * 255).astype(np.uint8)
66
 
67
+ colored = (cm.get_cmap("plasma")(d8)[:, :, :3] * 255).astype(np.uint8)
 
 
 
68
  return Image.fromarray(colored)
69
 
70
+
71
  def rgbd_to_pointcloud(rgb_pil: Image.Image, depth: np.ndarray):
72
  """Create an Open3D point cloud from RGB + relative depth."""
73
  # Normalize depth to 0..1 then to 0..255 uint8 for Open3D RGBD convenience
 
101
  pcd.orient_normals_to_align_with_direction()
102
  return pcd
103
 
104
+
105
  def pointcloud_to_mesh(pcd: o3d.geometry.PointCloud, depth=10):
106
  if len(pcd.points) == 0:
107
  return None
 
114
  mesh.compute_vertex_normals()
115
  return mesh
116
 
117
+
118
  def save_o3d(obj, path):
119
  ext = os.path.splitext(path)[1].lower()
120
  if isinstance(obj, o3d.geometry.PointCloud):
 
123
  else:
124
  raise ValueError("Point cloud: please save as .ply")
125
  elif isinstance(obj, o3d.geometry.TriangleMesh):
126
+ if ext in {".obj", ".ply"}:
 
 
127
  o3d.io.write_triangle_mesh(path, obj)
128
  else:
129
  raise ValueError("Mesh: use .obj or .ply")
130
  else:
131
  raise ValueError("Unsupported type for saving")
132
 
133
+
134
  def render_mesh_image(mesh: o3d.geometry.TriangleMesh, width=640, height=480):
135
  """
136
  Try offscreen render for a preview PNG. If it fails (e.g., no EGL/OSMesa),
 
139
  try:
140
  from open3d.visualization import rendering
141
 
142
+ # Ensure it has some color
143
  if not mesh.has_vertex_colors():
144
  mesh.paint_uniform_color([0.8, 0.8, 0.85])
145
 
 
156
  extent = bbox.get_extent()
157
  radius = np.linalg.norm(extent) * 0.8 + 1e-6
158
 
 
159
  cam = scene.camera
160
  cam.look_at(center, center + [0, 0, radius], [0, 1, 0])
161
 
 
165
  except Exception:
166
  return None
167
 
168
+
169
  # ------------------------------
170
  # Gradio pipeline
171
  # ------------------------------
 
210
 
211
  return depth_vis, preview, pcd_path, mesh_obj_path
212
 
213
+
214
  # ------------------------------
215
  # Interface
216
  # ------------------------------
217
  TITLE = "Monocular Depth β†’ Point Cloud β†’ Poisson Mesh (GLPN + Open3D)"
218
  DESC = """
219
  Upload an image. We estimate relative depth (GLPN), build a point cloud, and reconstruct
220
+ a mesh (Poisson). Outputs: depth preview, mesh preview (if renderer available),
221
+ and downloads for .ply (point cloud) and .obj (mesh).
222
  **Note:** monocular depth lacks absolute scale; this is for visualization/demo purposes.
223
  """
224
 
 
228
 
229
  with gr.Row():
230
  with gr.Column():
231
+ in_img = gr.Image(
232
+ type="pil",
233
+ sources=["upload", "clipboard"],
234
+ label="Input Image",
235
+ image_mode="RGB"
236
+ )
237
  poisson_depth = gr.Slider(5, 12, value=10, step=1, label="Poisson depth (mesh detail)")
238
  run_btn = gr.Button("Reconstruct 3D", variant="primary")
239
 
 
255
  mesh_obj_view.change(lambda p: p, inputs=mesh_obj_view, outputs=mesh_obj_file)
256
 
257
  if __name__ == "__main__":
258
+ # share=True creates a public link (useful on Spaces/Colab/local)
259
+ demo.launch(share=True)