mac9087 commited on
Commit
7432d4d
·
verified ·
1 Parent(s): cd6b776

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +745 -736
app.py CHANGED
@@ -1,762 +1,771 @@
1
- import os
2
- import torch
3
- import time
4
- import threading
5
- import json
6
- import gc
7
- from flask import Flask, request, jsonify, send_file, Response, stream_with_context
8
- from werkzeug.utils import secure_filename
9
- from PIL import Image
10
- import io
11
- import zipfile
12
- import uuid
13
- import traceback
14
- from huggingface_hub import snapshot_download, login
15
- from flask_cors import CORS
16
- import numpy as np
17
- import trimesh
18
- from transformers import pipeline, AutoImageProcessor, AutoModelForDepthEstimation
19
- from scipy.ndimage import gaussian_filter
20
- from scipy import interpolate
21
- import cv2
22
- from rembg import remove
23
 
24
- app = Flask(__name__)
25
- CORS(app)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
 
27
- # Configure directories
28
- UPLOAD_FOLDER = '/tmp/uploads'
29
- RESULTS_FOLDER = '/tmp/results'
30
- CACHE_DIR = '/tmp/huggingface'
31
- ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg'}
32
 
33
- os.makedirs(UPLOAD_FOLDER, exist_ok=True)
34
- os.makedirs(RESULTS_FOLDER, exist_ok=True)
35
- os.makedirs(CACHE_DIR, exist_ok=True)
 
 
36
 
37
- os.environ['HF_HOME'] = CACHE_DIR
38
- app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
39
- app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
40
 
41
- # Job tracking
42
- processing_jobs = {}
 
43
 
44
- # Model variables
45
- dpt_estimator = None
46
- depth_anything_model = None
47
- depth_anything_processor = None
48
- model_loaded = False
49
- model_loading = False
50
 
51
- TIMEOUT_SECONDS = 240
52
- MAX_DIMENSION = 518
 
 
 
 
53
 
54
- class TimeoutError(Exception):
55
- pass
56
 
57
- def process_with_timeout(function, args, timeout):
58
- result = [None]
59
- error = [None]
60
- completed = [False]
61
-
62
- def target():
63
- try:
64
- result[0] = function(*args)
65
- completed[0] = True
66
- except Exception as e:
67
- error[0] = e
68
-
69
- thread = threading.Thread(target=target)
70
- thread.daemon = True
71
- thread.start()
72
- thread.join(timeout)
73
-
74
- if not completed[0]:
75
- if thread.is_alive():
76
- return None, TimeoutError(f"Processing timed out after {timeout} seconds")
77
- elif error[0]:
78
- return None, error[0]
79
-
80
- if error[0]:
81
- return None, error[0]
82
-
83
- return result[0], None
84
 
85
- def allowed_file(filename):
86
- return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
 
88
- def remove_background(image_path):
89
- try:
90
- with open(image_path, "rb") as img_file:
91
- img_data = img_file.read()
92
- result = remove(img_data)
93
- img = Image.open(io.BytesIO(result)).convert("RGBA")
94
-
95
- # Check if image is fully transparent (no object)
96
- img_array = np.array(img)
97
- if np.all(img_array[:, :, 3] == 0):
98
- print(f"Warning: Image {image_path} is fully transparent or no object detected")
99
- return None
100
-
101
- # Create black background
102
- black_bg = Image.new("RGB", img.size, (0, 0, 0))
103
- black_bg.paste(img, (0, 0), img)
104
- return black_bg
105
- except Exception as e:
106
- print(f"Error in remove_background for {image_path}: {str(e)}")
107
- raise
108
 
109
- def preprocess_image(image_path):
110
- # Remove background and add black background
111
- img = remove_background(image_path)
112
- if img is None:
113
- raise ValueError("Image is fully transparent or no object detected")
114
-
115
- if img.width > MAX_DIMENSION or img.height > MAX_DIMENSION:
116
- if img.width > img.height:
117
- new_width = MAX_DIMENSION
118
- new_height = int(img.height * (MAX_DIMENSION / img.width))
119
- else:
120
- new_height = MAX_DIMENSION
121
- new_width = int(img.width * (MAX_DIMENSION / img.height))
122
- img = img.resize((new_width, new_height), Image.LANCZOS)
123
-
124
- img_array = np.array(img)
125
- if len(img_array.shape) == 3 and img_array.shape[2] == 3:
126
- lab = cv2.cvtColor(img_array, cv2.COLOR_RGB2LAB)
127
- l, a, b = cv2.split(lab)
128
- clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
129
- cl = clahe.apply(l)
130
- enhanced_lab = cv2.merge((cl, a, b))
131
- img_array = cv2.cvtColor(enhanced_lab, cv2.COLOR_LAB2RGB)
132
- img = Image.fromarray(img_array)
133
-
134
- return img
135
 
136
- def load_models():
137
- global dpt_estimator, depth_anything_model, depth_anything_processor, model_loaded, model_loading
138
-
139
- if model_loaded:
140
- return dpt_estimator, depth_anything_model, depth_anything_processor
141
-
142
- if model_loading:
143
- while model_loading and not model_loaded:
144
- time.sleep(0.5)
145
- return dpt_estimator, depth_anything_model, depth_anything_processor
146
-
147
- try:
148
- model_loading = True
149
- print("Loading models...")
150
-
151
- hf_token = os.environ.get('HF_TOKEN')
152
- if hf_token:
153
- login(token=hf_token)
154
- print("Authenticated with Hugging Face token")
155
-
156
- dpt_model_name = "Intel/dpt-large"
157
- max_retries = 3
158
- retry_delay = 5
159
- for attempt in range(max_retries):
160
- try:
161
- snapshot_download(
162
- repo_id=dpt_model_name,
163
- cache_dir=CACHE_DIR,
164
- resume_download=True,
165
- token=hf_token
166
- )
167
- break
168
- except Exception as e:
169
- if attempt < max_retries - 1:
170
- print(f"DPT download attempt {attempt+1} failed: {str(e)}. Retrying...")
171
- time.sleep(retry_delay)
172
- retry_delay *= 2
173
- else:
174
- raise
175
-
176
- dpt_estimator = pipeline(
177
- "depth-estimation",
178
- model=dpt_model_name,
179
- device=-1,
180
- cache_dir=CACHE_DIR,
181
- use_fast=True
182
- )
183
- print("DPT-Large loaded")
184
- gc.collect()
185
-
186
- da_model_name = "depth-anything/Depth-Anything-V2-Tiny-hf"
187
- for attempt in range(max_retries):
188
- try:
189
- snapshot_download(
190
- repo_id=da_model_name,
191
- cache_dir=CACHE_DIR,
192
- resume_download=True,
193
- token=hf_token
194
- )
195
- break
196
- except Exception as e:
197
- if attempt < max_retries - 1:
198
- print(f"Depth Anything download attempt {attempt+1} failed: {str(e)}. Retrying...")
199
- time.sleep(retry_delay)
200
- retry_delay *= 2
201
- else:
202
- print(f"Failed to load Depth Anything: {str(e)}. Falling back to DPT-Large only.")
203
- depth_anything_model = None
204
- depth_anything_processor = None
205
- model_loaded = True
206
- return dpt_estimator, None, None
207
-
208
- depth_anything_processor = AutoImageProcessor.from_pretrained(
209
- da_model_name,
210
- cache_dir=CACHE_DIR,
211
- token=hf_token
212
- )
213
- depth_anything_model = AutoModelForDepthEstimation.from_pretrained(
214
- da_model_name,
215
- cache_dir=CACHE_DIR,
216
- token=hf_token
217
- ).to("cpu")
218
-
219
- model_loaded = True
220
- print("Depth Anything loaded")
221
- return dpt_estimator, depth_anything_model, depth_anything_processor
222
-
223
- except Exception as e:
224
- print(f"Error loading models: {str(e)}")
225
- print(traceback.format_exc())
226
- raise
227
- finally:
228
- model_loading = False
229
 
230
- def fuse_depth_maps(dpt_depth, da_depth, detail_level='medium'):
231
- if isinstance(dpt_depth, Image.Image):
232
- dpt_depth = np.array(dpt_depth)
233
- if isinstance(da_depth, torch.Tensor):
234
- da_depth = da_depth.cpu().numpy()
235
- if len(dpt_depth.shape) > 2:
236
- dpt_depth = np.mean(dpt_depth, axis=2)
237
- if len(da_depth.shape) > 2:
238
- da_depth = np.mean(da_depth, axis=2)
239
-
240
- if dpt_depth.shape != da_depth.shape:
241
- da_depth = cv2.resize(da_depth, (dpt_depth.shape[1], dpt_depth.shape[0]), interpolation=cv2.INTER_CUBIC)
242
-
243
- p_low_dpt, p_high_dpt = np.percentile(dpt_depth, [1, 99])
244
- p_low_da, p_high_da = np.percentile(da_depth, [1, 99])
245
- dpt_depth = np.clip((dpt_depth - p_low_dpt) / (p_high_dpt - p_low_dpt), 0, 1) if p_high_dpt > p_low_dpt else dpt_depth
246
- da_depth = np.clip((da_depth - p_low_da) / (p_high_da - p_low_da), 0, 1) if p_high_da > p_low_da else da_depth
247
-
248
- if detail_level == 'high':
249
- weight_da = 0.7
250
- edges = cv2.Canny((da_depth * 255).astype(np.uint8), 50, 150)
251
- edge_mask = (edges > 0).astype(np.float32)
252
- dpt_weight = gaussian_filter(1 - edge_mask, sigma=1.0)
253
- da_weight = gaussian_filter(edge_mask, sigma=1.0)
254
- fused_depth = dpt_weight * dpt_depth + da_weight * da_depth * weight_da + (1 - weight_da) * dpt_depth
255
- else:
256
- weight_da = 0.5 if detail_level == 'medium' else 0.3
257
- fused_depth = (1 - weight_da) * dpt_depth + weight_da * da_depth
258
-
259
- fused_depth = np.clip(fused_depth, 0, 1)
260
- return fused_depth
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
261
 
262
- def enhance_depth_map(depth_map, detail_level='medium'):
263
- enhanced_depth = depth_map.copy().astype(np.float32)
264
- p_low, p_high = np.percentile(enhanced_depth, [1, 99])
265
- enhanced_depth = np.clip(enhanced_depth, p_low, p_high)
266
- enhanced_depth = (enhanced_depth - p_low) / (p_high - p_low) if p_high > p_low else enhanced_depth
267
-
268
- if detail_level == 'high':
269
- blurred = gaussian_filter(enhanced_depth, sigma=1.5)
270
- mask = enhanced_depth - blurred
271
- enhanced_depth = enhanced_depth + 1.5 * mask
272
- smooth1 = gaussian_filter(enhanced_depth, sigma=0.5)
273
- smooth2 = gaussian_filter(enhanced_depth, sigma=2.0)
274
- edge_mask = enhanced_depth - smooth2
275
- enhanced_depth = smooth1 + 1.2 * edge_mask
276
- elif detail_level == 'medium':
277
- blurred = gaussian_filter(enhanced_depth, sigma=1.0)
278
- mask = enhanced_depth - blurred
279
- enhanced_depth = enhanced_depth + 0.8 * mask
280
- enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.5)
281
- else:
282
- enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.7)
283
-
284
- enhanced_depth = np.clip(enhanced_depth, 0, 1)
285
- return enhanced_depth
 
 
 
 
 
 
 
286
 
287
- def depth_to_mesh(depth_map, image, resolution=80, detail_level='medium', view_angle=0):
288
- enhanced_depth = enhance_depth_map(depth_map, detail_level)
289
- h, w = enhanced_depth.shape
290
- x = np.linspace(0, w-1, resolution)
291
- y = np.linspace(0, h-1, resolution)
292
- x_grid, y_grid = np.meshgrid(x, y)
293
-
294
- interp_func = interpolate.RectBivariateSpline(
295
- np.arange(h), np.arange(w), enhanced_depth, kx=3, ky=3
296
- )
297
- z_values = interp_func(y, x, grid=True)
298
-
299
- if detail_level == 'high':
300
- dx = np.gradient(z_values, axis=1)
301
- dy = np.gradient(z_values, axis=0)
302
- gradient_magnitude = np.sqrt(dx**2 + dy**2)
303
- edge_mask = np.clip(gradient_magnitude * 5, 0, 0.2)
304
- z_values = z_values + edge_mask * (z_values - gaussian_filter(z_values, sigma=1.0))
305
-
306
- z_min, z_max = np.percentile(z_values, [2, 98])
307
- z_values = (z_values - z_min) / (z_max - z_min) if z_max > z_min else z_values
308
- z_scaling = 2.5 if detail_level == 'high' else 2.0 if detail_level == 'medium' else 1.5
309
- z_values = z_values * z_scaling
310
-
311
- x_grid = (x_grid / w - 0.5) * 2.0
312
- y_grid = (y_grid / h - 0.5) * 2.0
313
- vertices = np.vstack([x_grid.flatten(), -y_grid.flatten(), -z_values.flatten()]).T
314
-
315
- # Rotate vertices based on view angle (in radians)
316
- if view_angle != 0:
317
- rotation_matrix = trimesh.transformations.rotation_matrix(view_angle, [0, 1, 0])
318
- vertices = trimesh.transform_points(vertices, rotation_matrix)
319
-
320
- faces = []
321
- for i in range(resolution-1):
322
- for j in range(resolution-1):
323
- p1 = i * resolution + j
324
- p2 = i * resolution + (j + 1)
325
- p3 = (i + 1) * resolution + j
326
- p4 = (i + 1) * resolution + (j + 1)
327
- v1 = vertices[p1]
328
- v2 = vertices[p2]
329
- v3 = vertices[p3]
330
- v4 = vertices[p4]
331
- norm1 = np.cross(v2-v1, v4-v1)
332
- norm2 = np.cross(v4-v3, v1-v3)
333
- if np.dot(norm1, norm2) >= 0:
334
- faces.append([p1, p2, p4])
335
- faces.append([p1, p4, p3])
336
- else:
337
- faces.append([p1, p2, p3])
338
- faces.append([p2, p4, p3])
339
-
340
- faces = np.array(faces)
341
- mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
342
-
343
- if image:
344
- img_array = np.array(image)
345
- vertex_colors = np.zeros((vertices.shape[0], 4), dtype=np.uint8)
346
- for i in range(resolution):
347
- for j in range(resolution):
348
- img_x = j * (img_array.shape[1] - 1) / (resolution - 1)
349
- img_y = i * (img_array.shape[0] - 1) / (resolution - 1)
350
- x0, y0 = int(img_x), int(img_y)
351
- x1, y1 = min(x0 + 1, img_array.shape[1] - 1), min(y0 + 1, img_array.shape[0] - 1)
352
- wx = img_x - x0
353
- wy = img_y - y0
354
- vertex_idx = i * resolution + j
355
- if len(img_array.shape) == 3 and img_array.shape[2] == 3:
356
- r = int((1-wx)*(1-wy)*img_array[y0, x0, 0] + wx*(1-wy)*img_array[y0, x1, 0] +
357
- (1-wx)*wy*img_array[y1, x0, 0] + wx*wy*img_array[y1, x1, 0])
358
- g = int((1-wx)*(1-wy)*img_array[y0, x0, 1] + wx*(1-wy)*img_array[y0, x1, 1] +
359
- (1-wx)*wy*img_array[y1, x0, 1] + wx*wy*img_array[y1, x1, 1])
360
- b = int((1-wx)*(1-wy)*img_array[y0, x0, 2] + wx*(1-wy)*img_array[y0, x1, 2] +
361
- (1-wx)*wy*img_array[y1, x0, 2] + wx*wy*img_array[y1, x1, 2])
362
- vertex_colors[vertex_idx, :3] = [r, g, b]
363
- vertex_colors[vertex_idx, 3] = 255
364
- else:
365
- gray = int((1-wx)*(1-wy)*img_array[y0, x0] + wx*(1-wy)*img_array[y0, x1] +
366
- (1-wx)*wy*img_array[y1, x0] + wx*wy*img_array[y1, x1])
367
- vertex_colors[vertex_idx, :3] = [gray, gray, gray]
368
- vertex_colors[vertex_idx, 3] = 255
369
- mesh.visual.vertex_colors = vertex_colors
370
-
371
- if detail_level != 'high':
372
- mesh = mesh.smoothed(method='laplacian', iterations=1)
373
- mesh.fix_normals()
374
- return mesh
375
 
376
- def combine_meshes(meshes):
377
- if len(meshes) == 1:
378
- return meshes[0]
379
-
380
- combined_vertices = []
381
- combined_faces = []
382
- vertex_offset = 0
383
-
384
- for mesh in meshes:
385
- combined_vertices.append(mesh.vertices)
386
- combined_faces.append(mesh.faces + vertex_offset)
387
- vertex_offset += len(mesh.vertices)
388
-
389
- combined_vertices = np.vstack(combined_vertices)
390
- combined_faces = np.vstack(combined_faces)
391
-
392
- combined_mesh = trimesh.Trimesh(vertices=combined_vertices, faces=combined_faces)
393
-
394
- # Stitch overlapping vertices
395
- combined_mesh = combined_mesh.subdivide_to_size(max_edge=0.05)
396
- combined_mesh = combined_mesh.smoothed(method='laplacian', iterations=2)
397
-
398
- # Ensure watertight mesh
399
- combined_mesh.fill_holes()
400
- combined_mesh.fix_normals()
401
-
402
- return combined_mesh
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
403
 
404
- @app.route('/health', methods=['GET'])
405
- def health_check():
406
- return jsonify({
407
- "status": "healthy",
408
- "model": "DPT-Large + Depth Anything (Multi-View)",
409
- "device": "cpu"
410
- }), 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
411
 
412
- @app.route('/progress/<job_id>', methods=['GET'])
413
- def progress(job_id):
414
- def generate():
415
- if job_id not in processing_jobs:
416
- yield f"data: {json.dumps({'error': 'Job not found'})}\n\n"
417
- return
418
-
419
- job = processing_jobs[job_id]
420
- yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
421
-
422
- last_progress = job['progress']
423
- check_count = 0
424
- while job['status'] == 'processing':
425
- if job['progress'] != last_progress:
426
- yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
427
- last_progress = job['progress']
428
- time.sleep(0.5)
429
- check_count += 1
430
- if check_count > 60:
431
- if 'thread_alive' in job and not job['thread_alive']():
432
- job['status'] = 'error'
433
- job['error'] = 'Processing thread died unexpectedly'
434
- break
435
- check_count = 0
436
-
437
- if job['status'] == 'completed':
438
- yield f"data: {json.dumps({'status': 'completed', 'progress': 100, 'result_url': job['result_url'], 'preview_url': job['preview_url']})}\n\n"
439
- else:
440
- yield f"data: {json.dumps({'status': 'error', 'error': job['error']})}\n\n"
441
-
442
- return Response(stream_with_context(generate()), mimetype='text/event-stream')
443
 
444
- @app.route('/convert', methods=['POST'])
445
- def convert_image_to_3d():
446
- required_views = ['front', 'back']
447
- optional_views = ['left', 'right']
448
- view_files = {}
449
-
450
- for view in required_views + optional_views:
451
- if view in request.files and request.files[view].filename != '':
452
- view_files[view] = request.files[view]
453
-
454
- if not all(view in view_files for view in required_views):
455
- return jsonify({"error": "Front and back images are required"}), 400
456
-
457
- for view, file in view_files.items():
458
- if not allowed_file(file.filename):
459
- return jsonify({"error": f"File type not allowed for {view}. Supported types: {', '.join(ALLOWED_EXTENSIONS)}"}), 400
460
-
461
- try:
462
- mesh_resolution = min(int(request.form.get('mesh_resolution', 80)), 120)
463
- output_format = request.form.get('output_format', 'glb').lower()
464
- detail_level = request.form.get('detail_level', 'medium').lower()
465
- texture_quality = request.form.get('texture_quality', 'medium').lower()
466
- except ValueError:
467
- return jsonify({"error": "Invalid parameter values"}), 400
468
-
469
- if output_format not in ['obj', 'glb']:
470
- return jsonify({"error": "Unsupported output format. Use 'obj' or 'glb'"}), 400
471
-
472
- if detail_level == 'high':
473
- mesh_resolution = min(int(mesh_resolution * 1.5), 120)
474
- elif detail_level == 'low':
475
- mesh_resolution = max(int(mesh_resolution * 0.7), 50)
476
-
477
- job_id = str(uuid.uuid4())
478
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
479
- os.makedirs(output_dir, exist_ok=True)
480
-
481
- filepaths = {}
482
- for view, file in view_files.items():
483
- filename = secure_filename(file.filename)
484
- filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{view}_{filename}")
485
- file.save(filepath)
486
- filepaths[view] = filepath
487
-
488
- processing_jobs[job_id] = {
489
- 'status': 'processing',
490
- 'progress': 0,
491
- 'result_url': None,
492
- 'preview_url': None,
493
- 'error': None,
494
- 'output_format': output_format,
495
- 'created_at': time.time()
496
- }
497
-
498
- def process_images():
499
- thread = threading.current_thread()
500
- processing_jobs[job_id]['thread_alive'] = lambda: thread.is_alive()
501
-
502
- try:
503
- processing_jobs[job_id]['progress'] = 5
504
- images = {}
505
- for view, filepath in filepaths.items():
506
- try:
507
- images[view] = preprocess_image(filepath)
508
- except ValueError as e:
509
- processing_jobs[job_id]['status'] = 'error'
510
- processing_jobs[job_id]['error'] = f"Error preprocessing {view} image: {str(e)}"
511
- return
512
- processing_jobs[job_id]['progress'] = 10
513
-
514
- try:
515
- dpt_model, da_model, da_processor = load_models()
516
- processing_jobs[job_id]['progress'] = 20
517
- except Exception as e:
518
- processing_jobs[job_id]['status'] = 'error'
519
- processing_jobs[job_id]['error'] = f"Error loading models: {str(e)}"
520
- return
521
-
522
- try:
523
- def estimate_depths():
524
- meshes = []
525
- view_angles = {'front': 0, 'back': np.pi, 'left': np.pi/2, 'right': -np.pi/2}
526
- with torch.no_grad():
527
- for view, image in images.items():
528
- # DPT-Large
529
- dpt_result = dpt_model(image)
530
- dpt_depth = dpt_result["depth"]
531
-
532
- # Depth Anything (if loaded)
533
- if da_model and da_processor:
534
- inputs = da_processor(images=image, return_tensors="pt")
535
- inputs = {k: v.to("cpu") for k, v in inputs.items()}
536
- outputs = da_model(**inputs)
537
- da_depth = outputs.predicted_depth.squeeze()
538
- da_depth = torch.nn.functional.interpolate(
539
- da_depth.unsqueeze(0).unsqueeze(0),
540
- size=(image.height, image.width),
541
- mode='bicubic',
542
- align_corners=False
543
- ).squeeze()
544
- fused_depth = fuse_depth_maps(dpt_depth, da_depth, detail_level)
545
- else:
546
- fused_depth = np.array(dpt_depth) if isinstance(dpt_depth, Image.Image) else dpt_depth
547
- if len(fused_depth.shape) > 2:
548
- fused_depth = np.mean(fused_depth, axis=2)
549
- p_low, p_high = np.percentile(fused_depth, [1, 99])
550
- fused_depth = np.clip((fused_depth - p_low) / (p_high - p_low), 0, 1) if p_high > p_low else fused_depth
551
-
552
- mesh = depth_to_mesh(fused_depth, image, resolution=mesh_resolution, detail_level=detail_level, view_angle=view_angles[view])
553
- meshes.append(mesh)
554
- gc.collect()
555
-
556
- combined_mesh = combine_meshes(meshes)
557
- return combined_mesh
558
-
559
- combined_mesh, error = process_with_timeout(estimate_depths, [], TIMEOUT_SECONDS)
560
-
561
- if error:
562
- if isinstance(error, TimeoutError):
563
- processing_jobs[job_id]['status'] = 'error'
564
- processing_jobs[job_id]['error'] = f"Processing timed out after {TIMEOUT_SECONDS} seconds"
565
- return
566
- else:
567
- raise error
568
-
569
- processing_jobs[job_id]['progress'] = 80
570
-
571
- if output_format == 'obj':
572
- obj_path = os.path.join(output_dir, "model.obj")
573
- combined_mesh.export(
574
- obj_path,
575
- file_type='obj',
576
- include_normals=True,
577
- include_texture=True
578
- )
579
- zip_path = os.path.join(output_dir, "model.zip")
580
- with zipfile.ZipFile(zip_path, 'w') as zipf:
581
- zipf.write(obj_path, arcname="model.obj")
582
- mtl_path = os.path.join(output_dir, "model.mtl")
583
- if os.path.exists(mtl_path):
584
- zipf.write(mtl_path, arcname="model.mtl")
585
- texture_path = os.path.join(output_dir, "model.png")
586
- if os.path.exists(texture_path):
587
- zipf.write(texture_path, arcname="model.png")
588
-
589
- processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
590
- processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
591
-
592
- elif output_format == 'glb':
593
- glb_path = os.path.join(output_dir, "model.glb")
594
- combined_mesh.export(
595
- glb_path,
596
- file_type='glb'
597
- )
598
- processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
599
- processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
600
-
601
- processing_jobs[job_id]['status'] = 'completed'
602
- processing_jobs[job_id]['progress'] = 100
603
- print(f"Job {job_id} completed")
604
-
605
- except Exception as e:
606
- error_details = traceback.format_exc()
607
- processing_jobs[job_id]['status'] = 'error'
608
- processing_jobs[job_id]['error'] = f"Error during processing: {str(e)}"
609
- print(f"Error processing job {job_id}: {str(e)}")
610
- print(error_details)
611
- return
612
-
613
- for filepath in filepaths.values():
614
- if os.path.exists(filepath):
615
- os.remove(filepath)
616
- gc.collect()
617
-
618
- except Exception as e:
619
- error_details = traceback.format_exc()
620
- processing_jobs[job_id]['status'] = 'error'
621
- processing_jobs[job_id]['error'] = f"{str(e)}\n{error_details}"
622
- print(f"Error processing job {job_id}: {str(e)}")
623
- print(error_details)
624
- for filepath in filepaths.values():
625
- if os.path.exists(filepath):
626
- os.remove(filepath)
627
-
628
- processing_thread = threading.Thread(target=process_images)
629
- processing_thread.daemon = True
630
- processing_thread.start()
631
-
632
- return jsonify({"job_id": job_id}), 202
633
 
634
- @app.route('/download/<job_id>', methods=['GET'])
635
- def download_model(job_id):
636
- if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
637
- return jsonify({"error": "Model not found or processing not complete"}), 404
638
-
639
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
640
- output_format = processing_jobs[job_id].get('output_format', 'glb')
641
-
642
- if output_format == 'obj':
643
- zip_path = os.path.join(output_dir, "model.zip")
644
- if os.path.exists(zip_path):
645
- return send_file(zip_path, as_attachment=True, download_name="model.zip")
646
- else:
647
- glb_path = os.path.join(output_dir, "model.glb")
648
- if os.path.exists(glb_path):
649
- return send_file(glb_path, as_attachment=True, download_name="model.glb")
650
-
651
- return jsonify({"error": "File not found"}), 404
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
652
 
653
- @app.route('/preview/<job_id>', methods=['GET'])
654
- def preview_model(job_id):
655
- if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
656
- return jsonify({"error": "Model not found or processing not complete"}), 404
657
-
658
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
659
- output_format = processing_jobs[job_id].get('output_format', 'glb')
660
-
661
- if output_format == 'obj':
662
- obj_path = os.path.join(output_dir, "model.obj")
663
- if os.path.exists(obj_path):
664
- return send_file(obj_path, mimetype='model/obj')
665
- else:
666
- glb_path = os.path.join(output_dir, "model.glb")
667
- if os.path.exists(glb_path):
668
- return send_file(glb_path, mimetype='model/gltf-binary')
669
-
670
- return jsonify({"error": "File not found"}), 404
671
 
672
- def cleanup_old_jobs():
673
- current_time = time.time()
674
- job_ids_to_remove = []
675
-
676
- for job_id, job_data in processing_jobs.items():
677
- if job_data['status'] == 'completed' and (current_time - job_data.get('created_at', 0)) > 3600:
678
- job_ids_to_remove.append(job_id)
679
- elif job_data['status'] == 'error' and (current_time - job_data.get('created_at', 0)) > 1800:
680
- job_ids_to_remove.append(job_id)
681
-
682
- for job_id in job_ids_to_remove:
683
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
684
- try:
685
- import shutil
686
- if os.path.exists(output_dir):
687
- shutil.rmtree(output_dir)
688
- except Exception as e:
689
- print(f"Error cleaning up job {job_id}: {str(e)}")
690
-
691
- if job_id in processing_jobs:
692
- del processing_jobs[job_id]
693
-
694
- threading.Timer(300, cleanup_old_jobs).start()
695
 
696
- @app.route('/model-info/<job_id>', methods=['GET'])
697
- def model_info(job_id):
698
- if job_id not in processing_jobs:
699
- return jsonify({"error": "Model not found"}), 404
700
-
701
- job = processing_jobs[job_id]
702
-
703
- if job['status'] != 'completed':
704
- return jsonify({
705
- "status": job['status'],
706
- "progress": job['progress'],
707
- "error": job.get('error')
708
- }), 200
709
-
710
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
711
- model_stats = {}
712
-
713
- if job['output_format'] == 'obj':
714
- obj_path = os.path.join(output_dir, "model.obj")
715
- zip_path = os.path.join(output_dir, "model.zip")
716
- if os.path.exists(obj_path):
717
- model_stats['obj_size'] = os.path.getsize(obj_path)
718
- if os.path.exists(zip_path):
719
- model_stats['package_size'] = os.path.getsize(zip_path)
720
- else:
721
- glb_path = os.path.join(output_dir, "model.glb")
722
- if os.path.exists(glb_path):
723
- model_stats['model_size'] = os.path.getsize(glb_path)
724
-
725
- return jsonify({
726
- "status": job['status'],
727
- "model_format": job['output_format'],
728
- "download_url": job['result_url'],
729
- "preview_url": job['preview_url'],
730
- "model_stats": model_stats,
731
- "created_at": job.get('created_at'),
732
- "completed_at": job.get('completed_at')
733
- }), 200
734
 
735
- @app.route('/', methods=['GET'])
736
- def index():
737
- return jsonify({
738
- "message": "Multi-View Image to 3D API (DPT-Large + Depth Anything)",
739
- "endpoints": [
740
- "/convert",
741
- "/progress/<job_id>",
742
- "/download/<job_id>",
743
- "/preview/<job_id>",
744
- "/model-info/<job_id>"
745
- ],
746
- "parameters": {
747
- "front": "Image file (required)",
748
- "back": "Image file (required)",
749
- "left": "Image file (optional)",
750
- "right": "Image file (optional)",
751
- "mesh_resolution": "Integer (50-120)",
752
- "output_format": "obj or glb",
753
- "detail_level": "low, medium, or high",
754
- "texture_quality": "low, medium, or high"
755
- },
756
- "description": "Creates high-quality 3D models from multiple 2D images (front, back, left, right) using DPT-Large and Depth Anything."
757
- }), 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
758
 
759
- if __name__ == '__main__':
760
- cleanup_old_jobs()
761
- port = int(os.environ.get('PORT', 7860))
762
- app.run(host='0.0.0.0', port=port)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
 
2
+ import os
3
+ import torch
4
+ import time
5
+ import threading
6
+ import json
7
+ import gc
8
+ from flask import Flask, request, jsonify, send_file, Response, stream_with_context
9
+ from werkzeug.utils import secure_filename
10
+ from PIL import Image
11
+ import io
12
+ import zipfile
13
+ import uuid
14
+ import traceback
15
+ from huggingface_hub import snapshot_download, login
16
+ from flask_cors import CORS
17
+ import numpy as np
18
+ import trimesh
19
+ from transformers import pipeline, AutoImageProcessor, AutoModelForDepthEstimation
20
+ from scipy.ndimage import gaussian_filter
21
+ from scipy import interpolate
22
+ import cv2
23
+ from rembg import remove
24
 
25
+ app = Flask(__name__)
26
+ CORS(app)
 
 
 
27
 
28
+ # Configure directories
29
+ UPLOAD_FOLDER = '/tmp/uploads'
30
+ RESULTS_FOLDER = '/tmp/results'
31
+ CACHE_DIR = '/tmp/huggingface'
32
+ ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg'}
33
 
34
+ os.makedirs(UPLOAD_FOLDER, exist_ok=True)
35
+ os.makedirs(RESULTS_FOLDER, exist_ok=True)
36
+ os.makedirs(CACHE_DIR, exist_ok=True)
37
 
38
+ os.environ['HF_HOME'] = CACHE_DIR
39
+ app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
40
+ app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
41
 
42
+ # Job tracking
43
+ processing_jobs = {}
 
 
 
 
44
 
45
+ # Model variables
46
+ dpt_estimator = None
47
+ depth_anything_model = None
48
+ depth_anything_processor = None
49
+ model_loaded = False
50
+ model_loading = False
51
 
52
+ TIMEOUT_SECONDS = 240
53
+ MAX_DIMENSION = 518
54
 
55
+ class TimeoutError(Exception):
56
+ pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
 
58
+ def process_with_timeout(function, args, timeout):
59
+ result = [None]
60
+ error = [None]
61
+ completed = [False]
62
+
63
+ def target():
64
+ try:
65
+ result[0] = function(*args)
66
+ completed[0] = True
67
+ except Exception as e:
68
+ error[0] = e
69
+
70
+ thread = threading.Thread(target=target)
71
+ thread.daemon = True
72
+ thread.start()
73
+ thread.join(timeout)
74
+
75
+ if not completed[0]:
76
+ if thread.is_alive():
77
+ return None, TimeoutError(f"Processing timed out after {timeout} seconds")
78
+ elif error[0]:
79
+ return None, error[0]
80
+
81
+ if error[0]:
82
+ return None, error[0]
83
+
84
+ return result[0], None
85
 
86
+ def allowed_file(filename):
87
+ return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
89
+ def remove_background(image_path):
90
+ try:
91
+ with open(image_path, "rb") as img_file:
92
+ img_data = img_file.read()
93
+ result = remove(img_data)
94
+ img = Image.open(io.BytesIO(result)).convert("RGBA")
95
+
96
+ # Check if image is fully transparent (no object)
97
+ img_array = np.array(img)
98
+ if np.all(img_array[:, :, 3] == 0):
99
+ print(f"Warning: Image {image_path} is fully transparent or no object detected")
100
+ return None
101
+
102
+ # Create black background
103
+ black_bg = Image.new("RGB", img.size, (0, 0, 0))
104
+ black_bg.paste(img, (0, 0), img)
105
+ return black_bg
106
+ except Exception as e:
107
+ print(f"Error in remove_background for {image_path}: {str(e)}")
108
+ raise
 
 
 
 
 
 
109
 
110
+ def preprocess_image(image_path):
111
+ # Remove background and add black background
112
+ img = remove_background(image_path)
113
+ if img is None:
114
+ raise ValueError("Image is fully transparent or no object detected")
115
+
116
+ if img.width > MAX_DIMENSION or img.height > MAX_DIMENSION:
117
+ if img.width > img.height:
118
+ new_width = MAX_DIMENSION
119
+ new_height = int(img.height * (MAX_DIMENSION / img.width))
120
+ else:
121
+ new_height = MAX_DIMENSION
122
+ new_width = int(img.width * (MAX_DIMENSION / img.height))
123
+ img = img.resize((new_width, new_height), Image.LANCZOS)
124
+
125
+ img_array = np.array(img)
126
+ if len(img_array.shape) == 3 and img_array.shape[2] == 3:
127
+ lab = cv2.cvtColor(img_array, cv2.COLOR_RGB2LAB)
128
+ l, a, b = cv2.split(lab)
129
+ clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
130
+ cl = clahe.apply(l)
131
+ enhanced_lab = cv2.merge((cl, a, b))
132
+ img_array = cv2.cvtColor(enhanced_lab, cv2.COLOR_LAB2RGB)
133
+ img = Image.fromarray(img_array)
134
+
135
+ return img
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
 
137
+ def load_models():
138
+ global dpt_estimator, depth_anything_model, depth_anything_processor, model_loaded, model_loading
139
+
140
+ if model_loaded:
141
+ return dpt_estimator, depth_anything_model, depth_anything_processor
142
+
143
+ if model_loading:
144
+ while model_loading and not model_loaded:
145
+ time.sleep(0.5)
146
+ return dpt_estimator, depth_anything_model, depth_anything_processor
147
+
148
+ try:
149
+ model_loading = True
150
+ print("Loading models...")
151
+
152
+ hf_token = os.environ.get('HF_TOKEN')
153
+ if hf_token:
154
+ print("HF_TOKEN found, attempting login...")
155
+ login(token=hf_token)
156
+ print("Authenticated with Hugging Face token")
157
+ else:
158
+ print("Warning: HF_TOKEN not found in environment")
159
+
160
+ dpt_model_name = "Intel/dpt-large"
161
+ max_retries = 3
162
+ retry_delay = 5
163
+ for attempt in range(max_retries):
164
+ try:
165
+ print(f"Attempting to download {dpt_model_name}, attempt {attempt+1}")
166
+ snapshot_download(
167
+ repo_id=dpt_model_name,
168
+ cache_dir=CACHE_DIR,
169
+ resume_download=True,
170
+ token=hf_token
171
+ )
172
+ print(f"Successfully downloaded {dpt_model_name}")
173
+ break
174
+ except Exception as e:
175
+ if attempt < max_retries - 1:
176
+ print(f"DPT download attempt {attempt+1} failed: {str(e)}. Retrying after {retry_delay}s...")
177
+ time.sleep(retry_delay)
178
+ retry_delay *= 2
179
+ else:
180
+ raise
181
+
182
+ dpt_estimator = pipeline(
183
+ "depth-estimation",
184
+ model=dpt_model_name,
185
+ device=-1,
186
+ cache_dir=CACHE_DIR,
187
+ use_fast=True
188
+ )
189
+ print("DPT-Large loaded")
190
+ gc.collect()
191
+
192
+ da_model_name = "depth-anything/Depth-Anything-V2-Tiny-hf"
193
+ for attempt in range(max_retries):
194
+ try:
195
+ print(f"Attempting to download {da_model_name}, attempt {attempt+1}")
196
+ snapshot_download(
197
+ repo_id=da_model_name,
198
+ cache_dir=CACHE_DIR,
199
+ resume_download=True,
200
+ token=hf_token
201
+ )
202
+ print(f"Successfully downloaded {da_model_name}")
203
+ break
204
+ except Exception as e:
205
+ if attempt < max_retries - 1:
206
+ print(f"Depth Anything download attempt {attempt+1} failed: {str(e)}. Retrying after {retry_delay}s...")
207
+ time.sleep(retry_delay)
208
+ retry_delay *= 2
209
+ else:
210
+ print(f"Failed to load Depth Anything: {str(e)}. Falling back to DPT-Large only.")
211
+ depth_anything_model = None
212
+ depth_anything_processor = None
213
+ model_loaded = True
214
+ return dpt_estimator, None, None
215
+
216
+ depth_anything_processor = AutoImageProcessor.from_pretrained(
217
+ da_model_name,
218
+ cache_dir=CACHE_DIR,
219
+ token=hf_token
220
+ )
221
+ depth_anything_model = AutoModelForDepthEstimation.from_pretrained(
222
+ da_model_name,
223
+ cache_dir=CACHE_DIR,
224
+ token=hf_token
225
+ ).to("cpu")
226
+
227
+ model_loaded = True
228
+ print("Depth Anything loaded")
229
+ return dpt_estimator, depth_anything_model, depth_anything_processor
230
+
231
+ except Exception as e:
232
+ print(f"Error loading models: {str(e)}")
233
+ print(traceback.format_exc())
234
+ raise
235
+ finally:
236
+ model_loading = False
237
 
238
+ def fuse_depth_maps(dpt_depth, da_depth, detail_level='medium'):
239
+ if isinstance(dpt_depth, Image.Image):
240
+ dpt_depth = np.array(dpt_depth)
241
+ if isinstance(da_depth, torch.Tensor):
242
+ da_depth = da_depth.cpu().numpy()
243
+ if len(dpt_depth.shape) > 2:
244
+ dpt_depth = np.mean(dpt_depth, axis=2)
245
+ if len(da_depth.shape) > 2:
246
+ da_depth = np.mean(da_depth, axis=2)
247
+
248
+ if dpt_depth.shape != da_depth.shape:
249
+ da_depth = cv2.resize(da_depth, (dpt_depth.shape[1], dpt_depth.shape[0]), interpolation=cv2.INTER_CUBIC)
250
+
251
+ p_low_dpt, p_high_dpt = np.percentile(dpt_depth, [1, 99])
252
+ p_low_da, p_high_da = np.percentile(da_depth, [1, 99])
253
+ dpt_depth = np.clip((dpt_depth - p_low_dpt) / (p_high_dpt - p_low_dpt), 0, 1) if p_high_dpt > p_low_dpt else dpt_depth
254
+ da_depth = np.clip((da_depth - p_low_da) / (p_high_da - p_low_da), 0, 1) if p_high_da > p_low_da else da_depth
255
+
256
+ if detail_level == 'high':
257
+ weight_da = 0.7
258
+ edges = cv2.Canny((da_depth * 255).astype(np.uint8), 50, 150)
259
+ edge_mask = (edges > 0).astype(np.float32)
260
+ dpt_weight = gaussian_filter(1 - edge_mask, sigma=1.0)
261
+ da_weight = gaussian_filter(edge_mask, sigma=1.0)
262
+ fused_depth = dpt_weight * dpt_depth + da_weight * da_depth * weight_da + (1 - weight_da) * dpt_depth
263
+ else:
264
+ weight_da = 0.5 if detail_level == 'medium' else 0.3
265
+ fused_depth = (1 - weight_da) * dpt_depth + weight_da * da_depth
266
+
267
+ fused_depth = np.clip(fused_depth, 0, 1)
268
+ return fused_depth
269
 
270
+ def enhance_depth_map(depth_map, detail_level='medium'):
271
+ enhanced_depth = depth_map.copy().astype(np.float32)
272
+ p_low, p_high = np.percentile(enhanced_depth, [1, 99])
273
+ enhanced_depth = np.clip(enhanced_depth, p_low, p_high)
274
+ enhanced_depth = (enhanced_depth - p_low) / (p_high - p_low) if p_high > p_low else enhanced_depth
275
+
276
+ if detail_level == 'high':
277
+ blurred = gaussian_filter(enhanced_depth, sigma=1.5)
278
+ mask = enhanced_depth - blurred
279
+ enhanced_depth = enhanced_depth + 1.5 * mask
280
+ smooth1 = gaussian_filter(enhanced_depth, sigma=0.5)
281
+ smooth2 = gaussian_filter(enhanced_depth, sigma=2.0)
282
+ edge_mask = enhanced_depth - smooth2
283
+ enhanced_depth = smooth1 + 1.2 * edge_mask
284
+ elif detail_level == 'medium':
285
+ blurred = gaussian_filter(enhanced_depth, sigma=1.0)
286
+ mask = enhanced_depth - blurred
287
+ enhanced_depth = enhanced_depth + 0.8 * mask
288
+ enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.5)
289
+ else:
290
+ enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.7)
291
+
292
+ enhanced_depth = np.clip(enhanced_depth, 0, 1)
293
+ return enhanced_depth
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
294
 
295
+ def depth_to_mesh(depth_map, image, resolution=80, detail_level='medium', view_angle=0):
296
+ enhanced_depth = enhance_depth_map(depth_map, detail_level)
297
+ h, w = enhanced_depth.shape
298
+ x = np.linspace(0, w-1, resolution)
299
+ y = np.linspace(0, h-1, resolution)
300
+ x_grid, y_grid = np.meshgrid(x, y)
301
+
302
+ interp_func = interpolate.RectBivariateSpline(
303
+ np.arange(h), np.arange(w), enhanced_depth, kx=3, ky=3
304
+ )
305
+ z_values = interp_func(y, x, grid=True)
306
+
307
+ if detail_level == 'high':
308
+ dx = np.gradient(z_values, axis=1)
309
+ dy = np.gradient(z_values, axis=0)
310
+ gradient_magnitude = np.sqrt(dx**2 + dy**2)
311
+ edge_mask = np.clip(gradient_magnitude * 5, 0, 0.2)
312
+ z_values = z_values + edge_mask * (z_values - gaussian_filter(z_values, sigma=1.0))
313
+
314
+ z_min, z_max = np.percentile(z_values, [2, 98])
315
+ z_values = (z_values - z_min) / (z_max - z_min) if z_max > z_min else z_values
316
+ z_scaling = 2.5 if detail_level == 'high' else 2.0 if detail_level == 'medium' else 1.5
317
+ z_values = z_values * z_scaling
318
+
319
+ x_grid = (x_grid / w - 0.5) * 2.0
320
+ y_grid = (y_grid / h - 0.5) * 2.0
321
+ vertices = np.vstack([x_grid.flatten(), -y_grid.flatten(), -z_values.flatten()]).T
322
+
323
+ # Rotate vertices based on view angle (in radians)
324
+ if view_angle != 0:
325
+ rotation_matrix = trimesh.transformations.rotation_matrix(view_angle, [0, 1, 0])
326
+ vertices = trimesh.transform_points(vertices, rotation_matrix)
327
+
328
+ faces = []
329
+ for i in range(resolution-1):
330
+ for j in range(resolution-1):
331
+ p1 = i * resolution + j
332
+ p2 = i * resolution + (j + 1)
333
+ p3 = (i + 1) * resolution + j
334
+ p4 = (i + 1) * resolution + (j + 1)
335
+ v1 = vertices[p1]
336
+ v2 = vertices[p2]
337
+ v3 = vertices[p3]
338
+ v4 = vertices[p4]
339
+ norm1 = np.cross(v2-v1, v4-v1)
340
+ norm2 = np.cross(v4-v3, v1-v3)
341
+ if np.dot(norm1, norm2) >= 0:
342
+ faces.append([p1, p2, p4])
343
+ faces.append([p1, p4, p3])
344
+ else:
345
+ faces.append([p1, p2, p3])
346
+ faces.append([p2, p4, p3])
347
+
348
+ faces = np.array(faces)
349
+ mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
350
+
351
+ if image:
352
+ img_array = np.array(image)
353
+ vertex_colors = np.zeros((vertices.shape[0], 4), dtype=np.uint8)
354
+ for i in range(resolution):
355
+ for j in range(resolution):
356
+ img_x = j * (img_array.shape[1] - 1) / (resolution - 1)
357
+ img_y = i * (img_array.shape[0] - 1) / (resolution - 1)
358
+ x0, y0 = int(img_x), int(img_y)
359
+ x1, y1 = min(x0 + 1, img_array.shape[1] - 1), min(y0 + 1, img_array.shape[0] - 1)
360
+ wx = img_x - x0
361
+ wy = img_y - y0
362
+ vertex_idx = i * resolution + j
363
+ if len(img_array.shape) == 3 and img_array.shape[2] == 3:
364
+ r = int((1-wx)*(1-wy)*img_array[y0, x0, 0] + wx*(1-wy)*img_array[y0, x1, 0] +
365
+ (1-wx)*wy*img_array[y1, x0, 0] + wx*wy*img_array[y1, x1, 0])
366
+ g = int((1-wx)*(1-wy)*img_array[y0, x0, 1] + wx*(1-wy)*img_array[y0, x1, 1] +
367
+ (1-wx)*wy*img_array[y1, x0, 1] + wx*wy*img_array[y1, x1, 1])
368
+ b = int((1-wx)*(1-wy)*img_array[y0, x0, 2] + wx*(1-wy)*img_array[y0, x1, 2] +
369
+ (1-wx)*wy*img_array[y1, x0, 2] + wx*wy*img_array[y1, x1, 2])
370
+ vertex_colors[vertex_idx, :3] = [r, g, b]
371
+ vertex_colors[vertex_idx, 3] = 255
372
+ else:
373
+ gray = int((1-wx)*(1-wy)*img_array[y0, x0] + wx*(1-wy)*img_array[y0, x1] +
374
+ (1-wx)*wy*img_array[y1, x0] + wx*wy*img_array[y1, x1])
375
+ vertex_colors[vertex_idx, :3] = [gray, gray, gray]
376
+ vertex_colors[vertex_idx, 3] = 255
377
+ mesh.visual.vertex_colors = vertex_colors
378
+
379
+ if detail_level != 'high':
380
+ mesh = mesh.smoothed(method='laplacian', iterations=1)
381
+ mesh.fix_normals()
382
+ return mesh
383
 
384
+ def combine_meshes(meshes):
385
+ if len(meshes) == 1:
386
+ return meshes[0]
387
+
388
+ combined_vertices = []
389
+ combined_faces = []
390
+ vertex_offset = 0
391
+
392
+ for mesh in meshes:
393
+ combined_vertices.append(mesh.vertices)
394
+ combined_faces.append(mesh.faces + vertex_offset)
395
+ vertex_offset += len(mesh.vertices)
396
+
397
+ combined_vertices = np.vstack(combined_vertices)
398
+ combined_faces = np.vstack(combined_faces)
399
+
400
+ combined_mesh = trimesh.Trimesh(vertices=combined_vertices, faces=combined_faces)
401
+
402
+ # Stitch overlapping vertices
403
+ combined_mesh = combined_mesh.subdivide_to_size(max_edge=0.05)
404
+ combined_mesh = combined_mesh.smoothed(method='laplacian', iterations=2)
405
+
406
+ # Ensure watertight mesh
407
+ combined_mesh.fill_holes()
408
+ combined_mesh.fix_normals()
409
+
410
+ return combined_mesh
411
 
412
+ @app.route('/health', methods=['GET'])
413
+ def health_check():
414
+ return jsonify({
415
+ "status": "healthy",
416
+ "model": "DPT-Large + Depth Anything (Multi-View)",
417
+ "device": "cpu"
418
+ }), 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
419
 
420
+ @app.route('/progress/<job_id>', methods=['GET'])
421
+ def progress(job_id):
422
+ def generate():
423
+ if job_id not in processing_jobs:
424
+ yield f"data: {json.dumps({'error': 'Job not found'})}\n\n"
425
+ return
426
+
427
+ job = processing_jobs[job_id]
428
+ yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
429
+
430
+ last_progress = job['progress']
431
+ check_count = 0
432
+ while job['status'] == 'processing':
433
+ if job['progress'] != last_progress:
434
+ yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
435
+ last_progress = job['progress']
436
+ time.sleep(0.5)
437
+ check_count += 1
438
+ if check_count > 60:
439
+ if 'thread_alive' in job and not job['thread_alive']():
440
+ job['status'] = 'error'
441
+ job['error'] = 'Processing thread died unexpectedly'
442
+ break
443
+ check_count = 0
444
+
445
+ if job['status'] == 'completed':
446
+ yield f"data: {json.dumps({'status': 'completed', 'progress': 100, 'result_url': job['result_url'], 'preview_url': job['preview_url']})}\n\n"
447
+ else:
448
+ yield f"data: {json.dumps({'status': 'error', 'error': job['error']})}\n\n"
449
+
450
+ return Response(stream_with_context(generate()), mimetype='text/event-stream')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
451
 
452
+ @app.route('/convert', methods=['POST'])
453
+ def convert_image_to_3d():
454
+ required_views = ['front', 'back']
455
+ optional_views = ['left', 'right']
456
+ view_files = {}
457
+
458
+ for view in required_views + optional_views:
459
+ if view in request.files and request.files[view].filename != '':
460
+ view_files[view] = request.files[view]
461
+
462
+ if not all(view in view_files for view in required_views):
463
+ return jsonify({"error": "Front and back images are required"}), 400
464
+
465
+ for view, file in view_files.items():
466
+ if not allowed_file(file.filename):
467
+ return jsonify({"error": f"File type not allowed for {view}. Supported types: {', '.join(ALLOWED_EXTENSIONS)}"}), 400
468
+
469
+ try:
470
+ mesh_resolution = min(int(request.form.get('mesh_resolution', 80)), 120)
471
+ output_format = request.form.get('output_format', 'glb').lower()
472
+ detail_level = request.form.get('detail_level', 'medium').lower()
473
+ texture_quality = request.form.get('texture_quality', 'medium').lower()
474
+ except ValueError:
475
+ return jsonify({"error": "Invalid parameter values"}), 400
476
+
477
+ if output_format not in ['obj', 'glb']:
478
+ return jsonify({"error": "Unsupported output format. Use 'obj' or 'glb'"}), 400
479
+
480
+ if detail_level == 'high':
481
+ mesh_resolution = min(int(mesh_resolution * 1.5), 120)
482
+ elif detail_level == 'low':
483
+ mesh_resolution = max(int(mesh_resolution * 0.7), 50)
484
+
485
+ job_id = str(uuid.uuid4())
486
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
487
+ os.makedirs(output_dir, exist_ok=True)
488
+
489
+ filepaths = {}
490
+ for view, file in view_files.items():
491
+ filename = secure_filename(file.filename)
492
+ filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{view}_{filename}")
493
+ file.save(filepath)
494
+ filepaths[view] = filepath
495
+
496
+ processing_jobs[job_id] = {
497
+ 'status': 'processing',
498
+ 'progress': 0,
499
+ 'result_url': None,
500
+ 'preview_url': None,
501
+ 'error': None,
502
+ 'output_format': output_format,
503
+ 'created_at': time.time()
504
+ }
505
+
506
+ def process_images():
507
+ thread = threading.current_thread()
508
+ processing_jobs[job_id]['thread_alive'] = lambda: thread.is_alive()
509
+
510
+ try:
511
+ processing_jobs[job_id]['progress'] = 5
512
+ images = {}
513
+ for view, filepath in filepaths.items():
514
+ try:
515
+ images[view] = preprocess_image(filepath)
516
+ except ValueError as e:
517
+ processing_jobs[job_id]['status'] = 'error'
518
+ processing_jobs[job_id]['error'] = f"Error preprocessing {view} image: {str(e)}"
519
+ return
520
+ processing_jobs[job_id]['progress'] = 10
521
+
522
+ try:
523
+ dpt_model, da_model, da_processor = load_models()
524
+ processing_jobs[job_id]['progress'] = 20
525
+ except Exception as e:
526
+ processing_jobs[job_id]['status'] = 'error'
527
+ processing_jobs[job_id]['error'] = f"Error loading models: {str(e)}"
528
+ return
529
+
530
+ try:
531
+ def estimate_depths():
532
+ meshes = []
533
+ view_angles = {'front': 0, 'back': np.pi, 'left': np.pi/2, 'right': -np.pi/2}
534
+ with torch.no_grad():
535
+ for view, image in images.items():
536
+ # DPT-Large
537
+ dpt_result = dpt_model(image)
538
+ dpt_depth = dpt_result["depth"]
539
+
540
+ # Depth Anything (if loaded)
541
+ if da_model and da_processor:
542
+ inputs = da_processor(images=image, return_tensors="pt")
543
+ inputs = {k: v.to("cpu") for k, v in inputs.items()}
544
+ outputs = da_model(**inputs)
545
+ da_depth = outputs.predicted_depth.squeeze()
546
+ da_depth = torch.nn.functional.interpolate(
547
+ da_depth.unsqueeze(0).unsqueeze(0),
548
+ size=(image.height, image.width),
549
+ mode='bicubic',
550
+ align_corners=False
551
+ ).squeeze()
552
+ fused_depth = fuse_depth_maps(dpt_depth, da_depth, detail_level)
553
+ else:
554
+ fused_depth = np.array(dpt_depth) if isinstance(dpt_depth, Image.Image) else dpt_depth
555
+ if len(fused_depth.shape) > 2:
556
+ fused_depth = np.mean(fused_depth, axis=2)
557
+ p_low, p_high = np.percentile(fused_depth, [1, 99])
558
+ fused_depth = np.clip((fused_depth - p_low) / (p_high - p_low), 0, 1) if p_high > p_low else fused_depth
559
+
560
+ mesh = depth_to_mesh(fused_depth, image, resolution=mesh_resolution, detail_level=detail_level, view_angle=view_angles[view])
561
+ meshes.append(mesh)
562
+ gc.collect()
563
+
564
+ combined_mesh = combine_meshes(meshes)
565
+ return combined_mesh
566
+
567
+ combined_mesh, error = process_with_timeout(estimate_depths, [], TIMEOUT_SECONDS)
568
+
569
+ if error:
570
+ if isinstance(error, TimeoutError):
571
+ processing_jobs[job_id]['status'] = 'error'
572
+ processing_jobs[job_id]['error'] = f"Processing timed out after {TIMEOUT_SECONDS} seconds"
573
+ return
574
+ else:
575
+ raise error
576
+
577
+ processing_jobs[job_id]['progress'] = 80
578
+
579
+ if output_format == 'obj':
580
+ obj_path = os.path.join(output_dir, "model.obj")
581
+ combined_mesh.export(
582
+ obj_path,
583
+ file_type='obj',
584
+ include_normals=True,
585
+ include_texture=True
586
+ )
587
+ zip_path = os.path.join(output_dir, "model.zip")
588
+ with zipfile.ZipFile(zip_path, 'w') as zipf:
589
+ zipf.write(obj_path, arcname="model.obj")
590
+ mtl_path = os.path.join(output_dir, "model.mtl")
591
+ if os.path.exists(mtl_path):
592
+ zipf.write(mtl_path, arcname="model.mtl")
593
+ texture_path = os.path.join(output_dir, "model.png")
594
+ if os.path.exists(texture_path):
595
+ zipf.write(texture_path, arcname="model.png")
596
+
597
+ processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
598
+ processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
599
+
600
+ elif output_format == 'glb':
601
+ glb_path = os.path.join(output_dir, "model.glb")
602
+ combined_mesh.export(
603
+ glb_path,
604
+ file_type='glb'
605
+ )
606
+ processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
607
+ processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
608
+
609
+ processing_jobs[job_id]['status'] = 'completed'
610
+ processing_jobs[job_id]['progress'] = 100
611
+ print(f"Job {job_id} completed")
612
+
613
+ except Exception as e:
614
+ error_details = traceback.format_exc()
615
+ processing_jobs[job_id]['status'] = 'error'
616
+ processing_jobs[job_id]['error'] = f"Error during processing: {str(e)}"
617
+ print(f"Error processing job {job_id}: {str(e)}")
618
+ print(error_details)
619
+ return
620
+
621
+ for filepath in filepaths.values():
622
+ if os.path.exists(filepath):
623
+ os.remove(filepath)
624
+ gc.collect()
625
+
626
+ except Exception as e:
627
+ error_details = traceback.format_exc()
628
+ processing_jobs[job_id]['status'] = 'error'
629
+ processing_jobs[job_id]['error'] = f"{str(e)}\n{error_details}"
630
+ print(f"Error processing job {job_id}: {str(e)}")
631
+ print(error_details)
632
+ for filepath in filepaths.values():
633
+ if os.path.exists(filepath):
634
+ os.remove(filepath)
635
+
636
+ processing_thread = threading.Thread(target=process_images)
637
+ processing_thread.daemon = True
638
+ processing_thread.start()
639
+
640
+ return jsonify({"job_id": job_id}), 202
641
 
642
+ @app.route('/download/<job_id>', methods=['GET'])
643
+ def download_model(job_id):
644
+ if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
645
+ return jsonify({"error": "Model not found or processing not complete"}), 404
646
+
647
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
648
+ output_format = processing_jobs[job_id].get('output_format', 'glb')
649
+
650
+ if output_format == 'obj':
651
+ zip_path = os.path.join(output_dir, "model.zip")
652
+ if os.path.exists(zip_path):
653
+ return send_file(zip_path, as_attachment=True, download_name="model.zip")
654
+ else:
655
+ glb_path = os.path.join(output_dir, "model.glb")
656
+ if os.path.exists(glb_path):
657
+ return send_file(glb_path, as_attachment=True, download_name="model.glb")
658
+
659
+ return jsonify({"error": "File not found"}), 404
660
 
661
+ @app.route('/preview/<job_id>', methods=['GET'])
662
+ def preview_model(job_id):
663
+ if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
664
+ return jsonify({"error": "Model not found or processing not complete"}), 404
665
+
666
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
667
+ output_format = processing_jobs[job_id].get('output_format', 'glb')
668
+
669
+ if output_format == 'obj':
670
+ obj_path = os.path.join(output_dir, "model.obj")
671
+ if os.path.exists(obj_path):
672
+ return send_file(obj_path, mimetype='model/obj')
673
+ else:
674
+ glb_path = os.path.join(output_dir, "model.glb")
675
+ if os.path.exists(glb_path):
676
+ return send_file(glb_path, mimetype='model/gltf-binary')
677
+
678
+ return jsonify({"error": "File not found"}), 404
 
 
 
 
 
679
 
680
+ def cleanup_old_jobs():
681
+ current_time = time.time()
682
+ job_ids_to_remove = []
683
+
684
+ for job_id, job_data in processing_jobs.items():
685
+ if job_data['status'] == 'completed' and (current_time - job_data.get('created_at', 0)) > 3600:
686
+ job_ids_to_remove.append(job_id)
687
+ elif job_data['status'] == 'error' and (current_time - job_data.get('created_at', 0)) > 1800:
688
+ job_ids_to_remove.append(job_id)
689
+
690
+ for job_id in job_ids_to_remove:
691
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
692
+ try:
693
+ import shutil
694
+ if os.path.exists(output_dir):
695
+ shutil.rmtree(output_dir)
696
+ except Exception as e:
697
+ print(f"Error cleaning up job {job_id}: {str(e)}")
698
+
699
+ if job_id in processing_jobs:
700
+ del processing_jobs[job_id]
701
+
702
+ threading.Timer(300, cleanup_old_jobs).start()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
703
 
704
+ @app.route('/model-info/<job_id>', methods=['GET'])
705
+ def model_info(job_id):
706
+ if job_id not in processing_jobs:
707
+ return jsonify({"error": "Model not found"}), 404
708
+
709
+ job = processing_jobs[job_id]
710
+
711
+ if job['status'] != 'completed':
712
+ return jsonify({
713
+ "status": job['status'],
714
+ "progress": job['progress'],
715
+ "error": job.get('error')
716
+ }), 200
717
+
718
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
719
+ model_stats = {}
720
+
721
+ if job['output_format'] == 'obj':
722
+ obj_path = os.path.join(output_dir, "model.obj")
723
+ zip_path = os.path.join(output_dir, "model.zip")
724
+ if os.path.exists(obj_path):
725
+ model_stats['obj_size'] = os.path.getsize(obj_path)
726
+ if os.path.exists(zip_path):
727
+ model_stats['package_size'] = os.path.getsize(zip_path)
728
+ else:
729
+ glb_path = os.path.join(output_dir, "model.glb")
730
+ if os.path.exists(glb_path):
731
+ model_stats['model_size'] = os.path.getsize(glb_path)
732
+
733
+ return jsonify({
734
+ "status": job['status'],
735
+ "model_format": job['output_format'],
736
+ "download_url": job['result_url'],
737
+ "preview_url": job['preview_url'],
738
+ "model_stats": model_stats,
739
+ "created_at": job.get('created_at'),
740
+ "completed_at": job.get('completed_at')
741
+ }), 200
742
 
743
+ @app.route('/', methods=['GET'])
744
+ def index():
745
+ return jsonify({
746
+ "message": "Multi-View Image to 3D API (DPT-Large + Depth Anything)",
747
+ "endpoints": [
748
+ "/convert",
749
+ "/progress/<job_id>",
750
+ "/download/<job_id>",
751
+ "/preview/<job_id>",
752
+ "/model-info/<job_id>"
753
+ ],
754
+ "parameters": {
755
+ "front": "Image file (required)",
756
+ "back": "Image file (required)",
757
+ "left": "Image file (optional)",
758
+ "right": "Image file (optional)",
759
+ "mesh_resolution": "Integer (50-120)",
760
+ "output_format": "obj or glb",
761
+ "detail_level": "low, medium, or high",
762
+ "texture_quality": "low, medium, or high"
763
+ },
764
+ "description": "Creates high-quality 3D models from multiple 2D images (front, back, left, right) using DPT-Large and Depth Anything."
765
+ }), 200
766
+
767
+ if __name__ == '__main__':
768
+ cleanup_old_jobs()
769
+ port = int(os.environ.get('PORT', 7860))
770
+ app.run(host='0.0.0.0', port=port)
771
+