Spaces:
Running
on
Zero
Running
on
Zero
Update demo
Browse files
app.py
CHANGED
|
@@ -366,75 +366,356 @@ class FaceLiftPipeline:
|
|
| 366 |
raise gr.Error(f"Generation failed: {str(e)}")
|
| 367 |
|
| 368 |
# -----------------------------
|
| 369 |
-
#
|
|
|
|
| 370 |
# -----------------------------
|
| 371 |
GSPLAT_HEAD = """
|
| 372 |
-
<
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 382 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 383 |
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
container.appendChild(
|
| 388 |
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
camera.data.setPosition(0, 0, 2);
|
| 392 |
|
| 393 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 394 |
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
async function loadSplat(url) {
|
| 405 |
-
console.log("Loading splat from:", url);
|
| 406 |
-
ensureViewer();
|
| 407 |
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 411 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 412 |
|
| 413 |
-
|
| 414 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 416 |
try {
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
|
|
|
| 424 |
} catch (error) {
|
| 425 |
-
console.error(
|
| 426 |
}
|
| 427 |
}
|
| 428 |
|
| 429 |
-
|
| 430 |
-
window.__load_splat__ = loadSplat;
|
| 431 |
-
|
| 432 |
-
// Initialize viewer when DOM is ready
|
| 433 |
if (document.readyState === 'loading') {
|
| 434 |
-
document.addEventListener('DOMContentLoaded',
|
| 435 |
} else {
|
| 436 |
-
setTimeout(
|
| 437 |
}
|
|
|
|
| 438 |
</script>
|
| 439 |
"""
|
| 440 |
|
|
@@ -459,9 +740,9 @@ def main():
|
|
| 459 |
# After generation: copy ply into per-session folder and return viewer URL
|
| 460 |
def _prep_viewer_url(ply_path: str, session_id: str):
|
| 461 |
if not ply_path or not os.path.exists(ply_path):
|
| 462 |
-
return "Model file not found", ""
|
| 463 |
url = copy_to_session_and_get_url(ply_path, session_id)
|
| 464 |
-
return "
|
| 465 |
|
| 466 |
# Wrapper to return only the outputs we want to display
|
| 467 |
def _generate_and_filter_outputs(image_path, auto_crop, guidance_scale, random_seed, num_steps):
|
|
@@ -469,12 +750,12 @@ def main():
|
|
| 469 |
pipeline.generate_3d_head(image_path, auto_crop, guidance_scale, random_seed, num_steps)
|
| 470 |
return output_path, turntable_path, ply_path
|
| 471 |
|
| 472 |
-
gr.Markdown("## FaceLift: Single Image 3D Face Reconstruction
|
| 473 |
|
| 474 |
gr.Markdown("""
|
| 475 |
### 💡 Tips for Best Results
|
| 476 |
-
-
|
| 477 |
-
-
|
| 478 |
""")
|
| 479 |
|
| 480 |
with gr.Row():
|
|
@@ -491,15 +772,15 @@ def main():
|
|
| 491 |
gr.Examples(
|
| 492 |
examples=examples,
|
| 493 |
inputs=[in_image, auto_crop, guidance, seed, steps],
|
| 494 |
-
examples_per_page=
|
| 495 |
)
|
| 496 |
|
| 497 |
with gr.Column(scale=1):
|
| 498 |
-
gr.Markdown("### Interactive Gaussian Splat Viewer")
|
| 499 |
viewer = gr.HTML("<div id='splat-container' style='width:100%;height:600px;background:#000;border:1px solid #ccc;border-radius:8px;'></div>")
|
| 500 |
-
viewer_status = gr.Textbox(label="Viewer Status", value="
|
| 501 |
-
url_box = gr.Textbox(label="Model URL (for debugging)", interactive=False)
|
| 502 |
-
reload_btn = gr.Button("Reload Viewer", size="sm")
|
| 503 |
|
| 504 |
out_recon = gr.Image(label="3D Reconstruction")
|
| 505 |
out_video = gr.PlayableVideo(label="Turntable Animation")
|
|
|
|
| 366 |
raise gr.Error(f"Generation failed: {str(e)}")
|
| 367 |
|
| 368 |
# -----------------------------
|
| 369 |
+
# Custom WebGL Gaussian Splatting Viewer
|
| 370 |
+
# Based on antimatter15/splat viewer, modified for Gradio
|
| 371 |
# -----------------------------
|
| 372 |
GSPLAT_HEAD = """
|
| 373 |
+
<style>
|
| 374 |
+
#gs-canvas { width: 100%; height: 100%; display: block; cursor: grab; }
|
| 375 |
+
#gs-canvas:active { cursor: grabbing; }
|
| 376 |
+
</style>
|
| 377 |
+
<script>
|
| 378 |
+
// Custom WebGL Gaussian Splatting Viewer - Handles PLY files natively
|
| 379 |
+
(function() {
|
| 380 |
+
'use strict';
|
| 381 |
+
let canvas, gl, program, worker, texture, indexBuffer;
|
| 382 |
+
let viewMatrix, projectionMatrix;
|
| 383 |
+
let vertexCount = 0;
|
| 384 |
+
let isInitialized = false;
|
| 385 |
+
let u_view, u_projection, u_viewport, u_focal;
|
| 386 |
+
|
| 387 |
+
const vertexShaderSource = \`#version 300 es
|
| 388 |
+
precision highp float;
|
| 389 |
+
precision highp int;
|
| 390 |
+
uniform highp usampler2D u_texture;
|
| 391 |
+
uniform mat4 projection, view;
|
| 392 |
+
uniform vec2 focal, viewport;
|
| 393 |
+
in vec2 position;
|
| 394 |
+
in int index;
|
| 395 |
+
out vec4 vColor;
|
| 396 |
+
out vec2 vPosition;
|
| 397 |
+
|
| 398 |
+
void main () {
|
| 399 |
+
uvec4 cen = texelFetch(u_texture, ivec2((uint(index) & 0x3ffu) << 1, uint(index) >> 10), 0);
|
| 400 |
+
vec4 cam = view * vec4(uintBitsToFloat(cen.xyz), 1);
|
| 401 |
+
vec4 pos2d = projection * cam;
|
| 402 |
+
float clip = 1.2 * pos2d.w;
|
| 403 |
+
if (pos2d.z < -clip || pos2d.x < -clip || pos2d.x > clip || pos2d.y < -clip || pos2d.y > clip) {
|
| 404 |
+
gl_Position = vec4(0.0, 0.0, 2.0, 1.0);
|
| 405 |
+
return;
|
| 406 |
}
|
| 407 |
+
uvec4 cov = texelFetch(u_texture, ivec2(((uint(index) & 0x3ffu) << 1) | 1u, uint(index) >> 10), 0);
|
| 408 |
+
vec2 u1 = unpackHalf2x16(cov.x), u2 = unpackHalf2x16(cov.y), u3 = unpackHalf2x16(cov.z);
|
| 409 |
+
mat3 Vrk = mat3(u1.x, u1.y, u2.x, u1.y, u2.y, u3.x, u2.x, u3.x, u3.y);
|
| 410 |
+
mat3 J = mat3(focal.x / cam.z, 0., -(focal.x * cam.x) / (cam.z * cam.z),
|
| 411 |
+
0., -focal.y / cam.z, (focal.y * cam.y) / (cam.z * cam.z), 0., 0., 0.);
|
| 412 |
+
mat3 T = transpose(mat3(view)) * J;
|
| 413 |
+
mat3 cov2d = transpose(T) * Vrk * T;
|
| 414 |
+
float mid = (cov2d[0][0] + cov2d[1][1]) / 2.0;
|
| 415 |
+
float radius = length(vec2((cov2d[0][0] - cov2d[1][1]) / 2.0, cov2d[0][1]));
|
| 416 |
+
float lambda1 = mid + radius, lambda2 = mid - radius;
|
| 417 |
+
if(lambda2 < 0.0) return;
|
| 418 |
+
vec2 diagonalVector = normalize(vec2(cov2d[0][1], lambda1 - cov2d[0][0]));
|
| 419 |
+
vec2 majorAxis = min(sqrt(2.0 * lambda1), 1024.0) * diagonalVector;
|
| 420 |
+
vec2 minorAxis = min(sqrt(2.0 * lambda2), 1024.0) * vec2(diagonalVector.y, -diagonalVector.x);
|
| 421 |
+
vColor = clamp(pos2d.z/pos2d.w+1.0, 0.0, 1.0) * vec4((cov.w) & 0xffu, (cov.w >> 8) & 0xffu, (cov.w >> 16) & 0xffu, (cov.w >> 24) & 0xffu) / 255.0;
|
| 422 |
+
vPosition = position;
|
| 423 |
+
vec2 vCenter = vec2(pos2d) / pos2d.w;
|
| 424 |
+
gl_Position = vec4(vCenter + position.x * majorAxis / viewport + position.y * minorAxis / viewport, 0.0, 1.0);
|
| 425 |
+
}\`;
|
| 426 |
+
|
| 427 |
+
const fragmentShaderSource = \`#version 300 es
|
| 428 |
+
precision highp float;
|
| 429 |
+
in vec4 vColor;
|
| 430 |
+
in vec2 vPosition;
|
| 431 |
+
out vec4 fragColor;
|
| 432 |
+
void main () {
|
| 433 |
+
float A = -dot(vPosition, vPosition);
|
| 434 |
+
if (A < -4.0) discard;
|
| 435 |
+
float B = exp(A) * vColor.a;
|
| 436 |
+
fragColor = vec4(B * vColor.rgb, B);
|
| 437 |
+
}\`;
|
| 438 |
+
|
| 439 |
+
function createWorker() {
|
| 440 |
+
const code = \`
|
| 441 |
+
let buffer, vertexCount = 0;
|
| 442 |
+
const rowLength = 32;
|
| 443 |
+
var _floatView = new Float32Array(1), _int32View = new Int32Array(_floatView.buffer);
|
| 444 |
+
function floatToHalf(float) {
|
| 445 |
+
_floatView[0] = float; var f = _int32View[0];
|
| 446 |
+
var sign = (f >> 31) & 0x0001, exp = (f >> 23) & 0x00ff, frac = f & 0x007fffff, newExp;
|
| 447 |
+
if (exp == 0) newExp = 0;
|
| 448 |
+
else if (exp < 113) { newExp = 0; frac |= 0x00800000; frac = frac >> (113 - exp); if (frac & 0x01000000) { newExp = 1; frac = 0; }}
|
| 449 |
+
else if (exp < 142) newExp = exp - 112; else { newExp = 31; frac = 0; }
|
| 450 |
+
return (sign << 15) | (newExp << 10) | (frac >> 13);
|
| 451 |
+
}
|
| 452 |
+
function packHalf2x16(x, y) { return (floatToHalf(x) | (floatToHalf(y) << 16)) >>> 0; }
|
| 453 |
+
function processPlyBuffer(inputBuffer) {
|
| 454 |
+
const ubuf = new Uint8Array(inputBuffer);
|
| 455 |
+
const header = new TextDecoder().decode(ubuf.slice(0, 10240));
|
| 456 |
+
const header_end = "end_header\\\\n", header_end_index = header.indexOf(header_end);
|
| 457 |
+
if (header_end_index < 0) throw new Error("Unable to read .ply file header");
|
| 458 |
+
const vertexCount = parseInt(/element vertex (\\\\d+)\\\\n/.exec(header)[1]);
|
| 459 |
+
console.log("Vertex Count:", vertexCount);
|
| 460 |
+
let row_offset = 0, offsets = {}, types = {};
|
| 461 |
+
const TYPE_MAP = {double: "getFloat64", int: "getInt32", uint: "getUint32", float: "getFloat32", short: "getInt16", ushort: "getUint16", uchar: "getUint8"};
|
| 462 |
+
for (let prop of header.slice(0, header_end_index).split("\\\\n").filter(k => k.startsWith("property "))) {
|
| 463 |
+
const [p, type, name] = prop.split(" ");
|
| 464 |
+
const arrayType = TYPE_MAP[type] || "getInt8";
|
| 465 |
+
types[name] = arrayType; offsets[name] = row_offset;
|
| 466 |
+
row_offset += parseInt(arrayType.replace(/[^\\\\d]/g, "")) / 8;
|
| 467 |
+
}
|
| 468 |
+
let dataView = new DataView(inputBuffer, header_end_index + header_end.length), row = 0;
|
| 469 |
+
const attrs = new Proxy({}, {
|
| 470 |
+
get(target, prop) {
|
| 471 |
+
if (!types[prop]) throw new Error(prop + " not found");
|
| 472 |
+
return dataView[types[prop]](row * row_offset + offsets[prop], true);
|
| 473 |
+
}
|
| 474 |
+
});
|
| 475 |
+
let sizeList = new Float32Array(vertexCount), sizeIndex = new Uint32Array(vertexCount);
|
| 476 |
+
for (row = 0; row < vertexCount; row++) {
|
| 477 |
+
sizeIndex[row] = row;
|
| 478 |
+
if (!types["scale_0"]) continue;
|
| 479 |
+
const size = Math.exp(attrs.scale_0) * Math.exp(attrs.scale_1) * Math.exp(attrs.scale_2);
|
| 480 |
+
const opacity = 1 / (1 + Math.exp(-attrs.opacity));
|
| 481 |
+
sizeList[row] = size * opacity;
|
| 482 |
+
}
|
| 483 |
+
sizeIndex.sort((b, a) => sizeList[a] - sizeList[b]);
|
| 484 |
+
const buffer = new ArrayBuffer(rowLength * vertexCount);
|
| 485 |
+
for (let j = 0; j < vertexCount; j++) {
|
| 486 |
+
row = sizeIndex[j];
|
| 487 |
+
const position = new Float32Array(buffer, j * rowLength, 3);
|
| 488 |
+
const scales = new Float32Array(buffer, j * rowLength + 12, 3);
|
| 489 |
+
const rgba = new Uint8ClampedArray(buffer, j * rowLength + 24, 4);
|
| 490 |
+
const rot = new Uint8ClampedArray(buffer, j * rowLength + 28, 4);
|
| 491 |
+
if (types["scale_0"]) {
|
| 492 |
+
const qlen = Math.sqrt(attrs.rot_0 ** 2 + attrs.rot_1 ** 2 + attrs.rot_2 ** 2 + attrs.rot_3 ** 2);
|
| 493 |
+
rot[0] = (attrs.rot_0 / qlen) * 128 + 128; rot[1] = (attrs.rot_1 / qlen) * 128 + 128;
|
| 494 |
+
rot[2] = (attrs.rot_2 / qlen) * 128 + 128; rot[3] = (attrs.rot_3 / qlen) * 128 + 128;
|
| 495 |
+
scales[0] = Math.exp(attrs.scale_0); scales[1] = Math.exp(attrs.scale_1); scales[2] = Math.exp(attrs.scale_2);
|
| 496 |
+
} else { scales[0] = scales[1] = scales[2] = 0.01; rot[0] = 255; rot[1] = rot[2] = rot[3] = 0; }
|
| 497 |
+
position[0] = attrs.x; position[1] = attrs.y; position[2] = attrs.z;
|
| 498 |
+
if (types["f_dc_0"]) {
|
| 499 |
+
const SH_C0 = 0.28209479177387814;
|
| 500 |
+
rgba[0] = (0.5 + SH_C0 * attrs.f_dc_0) * 255; rgba[1] = (0.5 + SH_C0 * attrs.f_dc_1) * 255; rgba[2] = (0.5 + SH_C0 * attrs.f_dc_2) * 255;
|
| 501 |
+
} else { rgba[0] = attrs.red; rgba[1] = attrs.green; rgba[2] = attrs.blue; }
|
| 502 |
+
rgba[3] = types["opacity"] ? (1 / (1 + Math.exp(-attrs.opacity))) * 255 : 255;
|
| 503 |
+
}
|
| 504 |
+
return buffer;
|
| 505 |
+
}
|
| 506 |
+
function generateTexture() {
|
| 507 |
+
if (!buffer) return;
|
| 508 |
+
const f_buffer = new Float32Array(buffer), u_buffer = new Uint8Array(buffer);
|
| 509 |
+
const texwidth = 2048, texheight = Math.ceil((2 * vertexCount) / texwidth);
|
| 510 |
+
const texdata = new Uint32Array(texwidth * texheight * 4);
|
| 511 |
+
const texdata_c = new Uint8Array(texdata.buffer), texdata_f = new Float32Array(texdata.buffer);
|
| 512 |
+
for (let i = 0; i < vertexCount; i++) {
|
| 513 |
+
texdata_f[8 * i + 0] = f_buffer[8 * i + 0]; texdata_f[8 * i + 1] = f_buffer[8 * i + 1]; texdata_f[8 * i + 2] = f_buffer[8 * i + 2];
|
| 514 |
+
texdata_c[4 * (8 * i + 7) + 0] = u_buffer[32 * i + 24 + 0]; texdata_c[4 * (8 * i + 7) + 1] = u_buffer[32 * i + 24 + 1];
|
| 515 |
+
texdata_c[4 * (8 * i + 7) + 2] = u_buffer[32 * i + 24 + 2]; texdata_c[4 * (8 * i + 7) + 3] = u_buffer[32 * i + 24 + 3];
|
| 516 |
+
let scale = [f_buffer[8 * i + 3], f_buffer[8 * i + 4], f_buffer[8 * i + 5]];
|
| 517 |
+
let rot = [(u_buffer[32 * i + 28] - 128) / 128, (u_buffer[32 * i + 29] - 128) / 128, (u_buffer[32 * i + 30] - 128) / 128, (u_buffer[32 * i + 31] - 128) / 128];
|
| 518 |
+
const M = [1.0 - 2.0 * (rot[2] * rot[2] + rot[3] * rot[3]), 2.0 * (rot[1] * rot[2] + rot[0] * rot[3]), 2.0 * (rot[1] * rot[3] - rot[0] * rot[2]),
|
| 519 |
+
2.0 * (rot[1] * rot[2] - rot[0] * rot[3]), 1.0 - 2.0 * (rot[1] * rot[1] + rot[3] * rot[3]), 2.0 * (rot[2] * rot[3] + rot[0] * rot[1]),
|
| 520 |
+
2.0 * (rot[1] * rot[3] + rot[0] * rot[2]), 2.0 * (rot[2] * rot[3] - rot[0] * rot[1]), 1.0 - 2.0 * (rot[1] * rot[1] + rot[2] * rot[2])].map((k, i) => k * scale[Math.floor(i / 3)]);
|
| 521 |
+
const sigma = [M[0]*M[0]+M[3]*M[3]+M[6]*M[6], M[0]*M[1]+M[3]*M[4]+M[6]*M[7], M[0]*M[2]+M[3]*M[5]+M[6]*M[8],
|
| 522 |
+
M[1]*M[1]+M[4]*M[4]+M[7]*M[7], M[1]*M[2]+M[4]*M[5]+M[7]*M[8], M[2]*M[2]+M[5]*M[5]+M[8]*M[8]];
|
| 523 |
+
texdata[8 * i + 4] = packHalf2x16(4 * sigma[0], 4 * sigma[1]);
|
| 524 |
+
texdata[8 * i + 5] = packHalf2x16(4 * sigma[2], 4 * sigma[3]);
|
| 525 |
+
texdata[8 * i + 6] = packHalf2x16(4 * sigma[4], 4 * sigma[5]);
|
| 526 |
+
}
|
| 527 |
+
self.postMessage({ texdata, texwidth, texheight, vertexCount }, [texdata.buffer]);
|
| 528 |
+
}
|
| 529 |
+
self.onmessage = (e) => {
|
| 530 |
+
if (e.data.ply) {
|
| 531 |
+
buffer = processPlyBuffer(e.data.ply);
|
| 532 |
+
vertexCount = Math.floor(buffer.byteLength / rowLength);
|
| 533 |
+
generateTexture();
|
| 534 |
+
}
|
| 535 |
+
};
|
| 536 |
+
\`;
|
| 537 |
+
return new Worker(URL.createObjectURL(new Blob([code], { type: 'application/javascript' })));
|
| 538 |
+
}
|
| 539 |
+
|
| 540 |
+
const invert4 = (a) => {
|
| 541 |
+
let b00=a[0]*a[5]-a[1]*a[4],b01=a[0]*a[6]-a[2]*a[4],b02=a[0]*a[7]-a[3]*a[4],b03=a[1]*a[6]-a[2]*a[5],b04=a[1]*a[7]-a[3]*a[5],b05=a[2]*a[7]-a[3]*a[6];
|
| 542 |
+
let b06=a[8]*a[13]-a[9]*a[12],b07=a[8]*a[14]-a[10]*a[12],b08=a[8]*a[15]-a[11]*a[12],b09=a[9]*a[14]-a[10]*a[13],b10=a[9]*a[15]-a[11]*a[13],b11=a[10]*a[15]-a[11]*a[14];
|
| 543 |
+
let det=b00*b11-b01*b10+b02*b09+b03*b08-b04*b07+b05*b06;if(!det)return null;
|
| 544 |
+
return[(a[5]*b11-a[6]*b10+a[7]*b09)/det,(a[2]*b10-a[1]*b11-a[3]*b09)/det,(a[13]*b05-a[14]*b04+a[15]*b03)/det,(a[10]*b04-a[9]*b05-a[11]*b03)/det,
|
| 545 |
+
(a[6]*b08-a[4]*b11-a[7]*b07)/det,(a[0]*b11-a[2]*b08+a[3]*b07)/det,(a[14]*b02-a[12]*b05-a[15]*b01)/det,(a[8]*b05-a[10]*b02+a[11]*b01)/det,
|
| 546 |
+
(a[4]*b10-a[5]*b08+a[7]*b06)/det,(a[1]*b08-a[0]*b10-a[3]*b06)/det,(a[12]*b04-a[13]*b02+a[15]*b00)/det,(a[9]*b02-a[8]*b04-a[11]*b00)/det,
|
| 547 |
+
(a[5]*b07-a[4]*b09-a[6]*b06)/det,(a[0]*b09-a[1]*b07+a[2]*b06)/det,(a[13]*b01-a[12]*b03-a[14]*b00)/det,(a[8]*b03-a[9]*b01+a[10]*b00)/det];
|
| 548 |
+
};
|
| 549 |
+
const rotate4 = (a, rad, x, y, z) => {
|
| 550 |
+
let len = Math.hypot(x, y, z); x /= len; y /= len; z /= len;
|
| 551 |
+
let s = Math.sin(rad), c = Math.cos(rad), t = 1 - c;
|
| 552 |
+
let b00=x*x*t+c, b01=y*x*t+z*s, b02=z*x*t-y*s, b10=x*y*t-z*s, b11=y*y*t+c, b12=z*y*t+x*s, b20=x*z*t+y*s, b21=y*z*t-x*s, b22=z*z*t+c;
|
| 553 |
+
return [a[0]*b00+a[4]*b01+a[8]*b02,a[1]*b00+a[5]*b01+a[9]*b02,a[2]*b00+a[6]*b01+a[10]*b02,a[3]*b00+a[7]*b01+a[11]*b02,
|
| 554 |
+
a[0]*b10+a[4]*b11+a[8]*b12,a[1]*b10+a[5]*b11+a[9]*b12,a[2]*b10+a[6]*b11+a[10]*b12,a[3]*b10+a[7]*b11+a[11]*b12,
|
| 555 |
+
a[0]*b20+a[4]*b21+a[8]*b22,a[1]*b20+a[5]*b21+a[9]*b22,a[2]*b20+a[6]*b21+a[10]*b22,a[3]*b20+a[7]*b21+a[11]*b22,...a.slice(12,16)];
|
| 556 |
+
};
|
| 557 |
+
const translate4 = (a, x, y, z) => [...a.slice(0,12), a[0]*x+a[4]*y+a[8]*z+a[12], a[1]*x+a[5]*y+a[9]*z+a[13], a[2]*x+a[6]*y+a[10]*z+a[14], a[3]*x+a[7]*y+a[11]*z+a[15]];
|
| 558 |
+
const getProjectionMatrix = (fx, fy, width, height) => {
|
| 559 |
+
const znear = 0.2, zfar = 200;
|
| 560 |
+
return [(2*fx)/width, 0, 0, 0, 0, -(2*fy)/height, 0, 0, 0, 0, zfar/(zfar-znear), 1, 0, 0, -(zfar*znear)/(zfar-znear), 0];
|
| 561 |
+
};
|
| 562 |
+
|
| 563 |
+
function initViewer() {
|
| 564 |
+
if (isInitialized) return;
|
| 565 |
+
const container = document.getElementById('splat-container');
|
| 566 |
+
if (!container) { console.error('Container not found'); setTimeout(initViewer, 100); return; }
|
| 567 |
|
| 568 |
+
canvas = document.createElement('canvas');
|
| 569 |
+
canvas.id = 'gs-canvas';
|
| 570 |
+
container.innerHTML = '';
|
| 571 |
+
container.appendChild(canvas);
|
| 572 |
|
| 573 |
+
gl = canvas.getContext('webgl2', { antialias: false });
|
| 574 |
+
if (!gl) { console.error('WebGL2 not supported'); return; }
|
|
|
|
| 575 |
|
| 576 |
+
const vShader = gl.createShader(gl.VERTEX_SHADER);
|
| 577 |
+
gl.shaderSource(vShader, vertexShaderSource);
|
| 578 |
+
gl.compileShader(vShader);
|
| 579 |
+
const fShader = gl.createShader(gl.FRAGMENT_SHADER);
|
| 580 |
+
gl.shaderSource(fShader, fragmentShaderSource);
|
| 581 |
+
gl.compileShader(fShader);
|
| 582 |
+
program = gl.createProgram();
|
| 583 |
+
gl.attachShader(program, vShader);
|
| 584 |
+
gl.attachShader(program, fShader);
|
| 585 |
+
gl.linkProgram(program);
|
| 586 |
+
gl.useProgram(program);
|
| 587 |
|
| 588 |
+
gl.disable(gl.DEPTH_TEST);
|
| 589 |
+
gl.enable(gl.BLEND);
|
| 590 |
+
gl.blendFuncSeparate(gl.ONE_MINUS_DST_ALPHA, gl.ONE, gl.ONE_MINUS_DST_ALPHA, gl.ONE);
|
| 591 |
+
|
| 592 |
+
u_projection = gl.getUniformLocation(program, 'projection');
|
| 593 |
+
u_viewport = gl.getUniformLocation(program, 'viewport');
|
| 594 |
+
u_focal = gl.getUniformLocation(program, 'focal');
|
| 595 |
+
u_view = gl.getUniformLocation(program, 'view');
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
|
| 597 |
+
const vBuffer = gl.createBuffer();
|
| 598 |
+
gl.bindBuffer(gl.ARRAY_BUFFER, vBuffer);
|
| 599 |
+
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array([-2, -2, 2, -2, 2, 2, -2, 2]), gl.STATIC_DRAW);
|
| 600 |
+
const a_position = gl.getAttribLocation(program, 'position');
|
| 601 |
+
gl.enableVertexAttribArray(a_position);
|
| 602 |
+
gl.vertexAttribPointer(a_position, 2, gl.FLOAT, false, 0, 0);
|
| 603 |
+
|
| 604 |
+
texture = gl.createTexture();
|
| 605 |
+
gl.bindTexture(gl.TEXTURE_2D, texture);
|
| 606 |
+
gl.uniform1i(gl.getUniformLocation(program, 'u_texture'), 0);
|
| 607 |
+
|
| 608 |
+
indexBuffer = gl.createBuffer();
|
| 609 |
+
const a_index = gl.getAttribLocation(program, 'index');
|
| 610 |
+
gl.enableVertexAttribArray(a_index);
|
| 611 |
+
gl.bindBuffer(gl.ARRAY_BUFFER, indexBuffer);
|
| 612 |
+
gl.vertexAttribIPointer(a_index, 1, gl.INT, false, 0, 0);
|
| 613 |
+
gl.vertexAttribDivisor(a_index, 1);
|
| 614 |
+
|
| 615 |
+
viewMatrix = [0.99, 0, 0.14, 0, 0, 1, 0, 0, -0.14, 0, 0.99, 0, 0, 0, 2.5, 1];
|
| 616 |
+
const fx = 500, fy = 500;
|
| 617 |
+
|
| 618 |
+
function resize() {
|
| 619 |
+
const dpr = window.devicePixelRatio || 1;
|
| 620 |
+
const rect = container.getBoundingClientRect();
|
| 621 |
+
canvas.width = rect.width * dpr;
|
| 622 |
+
canvas.height = rect.height * dpr;
|
| 623 |
+
canvas.style.width = rect.width + 'px';
|
| 624 |
+
canvas.style.height = rect.height + 'px';
|
| 625 |
+
projectionMatrix = getProjectionMatrix(fx, fy, canvas.width, canvas.height);
|
| 626 |
+
gl.viewport(0, 0, canvas.width, canvas.height);
|
| 627 |
+
gl.uniformMatrix4fv(u_projection, false, projectionMatrix);
|
| 628 |
+
gl.uniform2fv(u_viewport, [canvas.width, canvas.height]);
|
| 629 |
+
gl.uniform2fv(u_focal, [fx, fy]);
|
| 630 |
}
|
| 631 |
+
window.addEventListener('resize', resize);
|
| 632 |
+
resize();
|
| 633 |
+
|
| 634 |
+
let startX, startY, down;
|
| 635 |
+
canvas.addEventListener('mousedown', (e) => { e.preventDefault(); startX = e.clientX; startY = e.clientY; down = e.ctrlKey || e.metaKey ? 2 : 1; });
|
| 636 |
+
canvas.addEventListener('mousemove', (e) => {
|
| 637 |
+
e.preventDefault();
|
| 638 |
+
if (down == 1) {
|
| 639 |
+
let inv = invert4(viewMatrix), dx = (5 * (e.clientX - startX)) / window.innerWidth, dy = (5 * (e.clientY - startY)) / window.innerHeight;
|
| 640 |
+
inv = translate4(inv, 0, 0, 2);
|
| 641 |
+
inv = rotate4(inv, dx, 0, 1, 0);
|
| 642 |
+
inv = rotate4(inv, -dy, 1, 0, 0);
|
| 643 |
+
inv = translate4(inv, 0, 0, -2);
|
| 644 |
+
viewMatrix = invert4(inv);
|
| 645 |
+
startX = e.clientX; startY = e.clientY;
|
| 646 |
+
} else if (down == 2) {
|
| 647 |
+
let inv = invert4(viewMatrix);
|
| 648 |
+
inv = translate4(inv, (-5 * (e.clientX - startX)) / window.innerWidth, 0, (5 * (e.clientY - startY)) / window.innerHeight);
|
| 649 |
+
viewMatrix = invert4(inv);
|
| 650 |
+
startX = e.clientX; startY = e.clientY;
|
| 651 |
+
}
|
| 652 |
+
});
|
| 653 |
+
canvas.addEventListener('mouseup', () => { down = false; });
|
| 654 |
+
canvas.addEventListener('wheel', (e) => {
|
| 655 |
+
e.preventDefault();
|
| 656 |
+
let inv = invert4(viewMatrix);
|
| 657 |
+
inv = translate4(inv, 0, 0, -e.deltaY * 0.001);
|
| 658 |
+
viewMatrix = invert4(inv);
|
| 659 |
+
}, { passive: false });
|
| 660 |
|
| 661 |
+
function render() {
|
| 662 |
+
if (vertexCount > 0) {
|
| 663 |
+
gl.uniformMatrix4fv(u_view, false, viewMatrix);
|
| 664 |
+
gl.clear(gl.COLOR_BUFFER_BIT);
|
| 665 |
+
gl.drawArraysInstanced(gl.TRIANGLE_FAN, 0, 4, vertexCount);
|
| 666 |
+
} else {
|
| 667 |
+
gl.clearColor(0, 0, 0, 1);
|
| 668 |
+
gl.clear(gl.COLOR_BUFFER_BIT);
|
| 669 |
+
}
|
| 670 |
+
requestAnimationFrame(render);
|
| 671 |
+
}
|
| 672 |
+
render();
|
| 673 |
|
| 674 |
+
worker = createWorker();
|
| 675 |
+
worker.onmessage = (e) => {
|
| 676 |
+
if (e.data.texdata) {
|
| 677 |
+
const { texdata, texwidth, texheight, vertexCount: vc } = e.data;
|
| 678 |
+
vertexCount = vc;
|
| 679 |
+
gl.bindTexture(gl.TEXTURE_2D, texture);
|
| 680 |
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
| 681 |
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
| 682 |
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
|
| 683 |
+
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
|
| 684 |
+
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA32UI, texwidth, texheight, 0, gl.RGBA_INTEGER, gl.UNSIGNED_INT, texdata);
|
| 685 |
+
const depthIndex = new Uint32Array(vc);
|
| 686 |
+
for (let i = 0; i < vc; i++) depthIndex[i] = i;
|
| 687 |
+
gl.bindBuffer(gl.ARRAY_BUFFER, indexBuffer);
|
| 688 |
+
gl.bufferData(gl.ARRAY_BUFFER, depthIndex, gl.STATIC_DRAW);
|
| 689 |
+
console.log('Loaded', vc, 'gaussians');
|
| 690 |
+
}
|
| 691 |
+
};
|
| 692 |
+
|
| 693 |
+
isInitialized = true;
|
| 694 |
+
console.log('Viewer initialized');
|
| 695 |
+
}
|
| 696 |
+
|
| 697 |
+
async function loadPlyFile(url) {
|
| 698 |
try {
|
| 699 |
+
console.log('Loading PLY from:', url);
|
| 700 |
+
if (!isInitialized) initViewer();
|
| 701 |
+
if (!worker) { console.error('Worker not initialized'); return; }
|
| 702 |
+
const response = await fetch(url);
|
| 703 |
+
if (!response.ok) throw new Error('Failed to load: ' + response.status);
|
| 704 |
+
const arrayBuffer = await response.arrayBuffer();
|
| 705 |
+
worker.postMessage({ ply: arrayBuffer }, [arrayBuffer]);
|
| 706 |
+
console.log('PLY file sent to worker');
|
| 707 |
} catch (error) {
|
| 708 |
+
console.error('Error loading PLY:', error);
|
| 709 |
}
|
| 710 |
}
|
| 711 |
|
| 712 |
+
window.__load_splat__ = loadPlyFile;
|
|
|
|
|
|
|
|
|
|
| 713 |
if (document.readyState === 'loading') {
|
| 714 |
+
document.addEventListener('DOMContentLoaded', () => setTimeout(initViewer, 100));
|
| 715 |
} else {
|
| 716 |
+
setTimeout(initViewer, 100);
|
| 717 |
}
|
| 718 |
+
})();
|
| 719 |
</script>
|
| 720 |
"""
|
| 721 |
|
|
|
|
| 740 |
# After generation: copy ply into per-session folder and return viewer URL
|
| 741 |
def _prep_viewer_url(ply_path: str, session_id: str):
|
| 742 |
if not ply_path or not os.path.exists(ply_path):
|
| 743 |
+
return "✗ Model file not found", ""
|
| 744 |
url = copy_to_session_and_get_url(ply_path, session_id)
|
| 745 |
+
return "✓ Model loaded! Use mouse to interact.", url
|
| 746 |
|
| 747 |
# Wrapper to return only the outputs we want to display
|
| 748 |
def _generate_and_filter_outputs(image_path, auto_crop, guidance_scale, random_seed, num_steps):
|
|
|
|
| 750 |
pipeline.generate_3d_head(image_path, auto_crop, guidance_scale, random_seed, num_steps)
|
| 751 |
return output_path, turntable_path, ply_path
|
| 752 |
|
| 753 |
+
gr.Markdown("## FaceLift: Single Image 3D Face Reconstruction.")
|
| 754 |
|
| 755 |
gr.Markdown("""
|
| 756 |
### 💡 Tips for Best Results
|
| 757 |
+
- Works best with near-frontal portrait images
|
| 758 |
+
- The provided checkpoints were not trained with accessories (glasses, hats, etc.). Portraits containing accessories may produce suboptimal results.
|
| 759 |
""")
|
| 760 |
|
| 761 |
with gr.Row():
|
|
|
|
| 772 |
gr.Examples(
|
| 773 |
examples=examples,
|
| 774 |
inputs=[in_image, auto_crop, guidance, seed, steps],
|
| 775 |
+
examples_per_page=10,
|
| 776 |
)
|
| 777 |
|
| 778 |
with gr.Column(scale=1):
|
| 779 |
+
gr.Markdown("### Interactive Gaussian Splat Viewer\n*Drag to rotate, Ctrl+drag to pan, scroll to zoom*")
|
| 780 |
viewer = gr.HTML("<div id='splat-container' style='width:100%;height:600px;background:#000;border:1px solid #ccc;border-radius:8px;'></div>")
|
| 781 |
+
viewer_status = gr.Textbox(label="Viewer Status", value="Viewer ready. Generate a model to view.", interactive=False)
|
| 782 |
+
url_box = gr.Textbox(label="Model URL (for debugging)", interactive=False, visible=False)
|
| 783 |
+
reload_btn = gr.Button("Reload Viewer", size="sm", visible=False)
|
| 784 |
|
| 785 |
out_recon = gr.Image(label="3D Reconstruction")
|
| 786 |
out_video = gr.PlayableVideo(label="Turntable Animation")
|