mac9087 commited on
Commit
ecde90f
·
verified ·
1 Parent(s): 7432d4d

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +743 -745
app.py CHANGED
@@ -1,771 +1,769 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
 
2
- import os
3
- import torch
4
- import time
5
- import threading
6
- import json
7
- import gc
8
- from flask import Flask, request, jsonify, send_file, Response, stream_with_context
9
- from werkzeug.utils import secure_filename
10
- from PIL import Image
11
- import io
12
- import zipfile
13
- import uuid
14
- import traceback
15
- from huggingface_hub import snapshot_download, login
16
- from flask_cors import CORS
17
- import numpy as np
18
- import trimesh
19
- from transformers import pipeline, AutoImageProcessor, AutoModelForDepthEstimation
20
- from scipy.ndimage import gaussian_filter
21
- from scipy import interpolate
22
- import cv2
23
- from rembg import remove
24
 
25
- app = Flask(__name__)
26
- CORS(app)
 
 
 
27
 
28
- # Configure directories
29
- UPLOAD_FOLDER = '/tmp/uploads'
30
- RESULTS_FOLDER = '/tmp/results'
31
- CACHE_DIR = '/tmp/huggingface'
32
- ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg'}
33
 
34
- os.makedirs(UPLOAD_FOLDER, exist_ok=True)
35
- os.makedirs(RESULTS_FOLDER, exist_ok=True)
36
- os.makedirs(CACHE_DIR, exist_ok=True)
37
 
38
- os.environ['HF_HOME'] = CACHE_DIR
39
- app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
40
- app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
41
 
42
- # Job tracking
43
- processing_jobs = {}
 
 
 
 
44
 
45
- # Model variables
46
- dpt_estimator = None
47
- depth_anything_model = None
48
- depth_anything_processor = None
49
- model_loaded = False
50
- model_loading = False
51
 
52
- TIMEOUT_SECONDS = 240
53
- MAX_DIMENSION = 518
54
 
55
- class TimeoutError(Exception):
56
- pass
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
 
58
- def process_with_timeout(function, args, timeout):
59
- result = [None]
60
- error = [None]
61
- completed = [False]
62
-
63
- def target():
64
- try:
65
- result[0] = function(*args)
66
- completed[0] = True
67
- except Exception as e:
68
- error[0] = e
69
-
70
- thread = threading.Thread(target=target)
71
- thread.daemon = True
72
- thread.start()
73
- thread.join(timeout)
74
-
75
- if not completed[0]:
76
- if thread.is_alive():
77
- return None, TimeoutError(f"Processing timed out after {timeout} seconds")
78
- elif error[0]:
79
- return None, error[0]
80
-
81
- if error[0]:
82
- return None, error[0]
83
-
84
- return result[0], None
85
 
86
- def allowed_file(filename):
87
- return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
 
89
- def remove_background(image_path):
90
- try:
91
- with open(image_path, "rb") as img_file:
92
- img_data = img_file.read()
93
- result = remove(img_data)
94
- img = Image.open(io.BytesIO(result)).convert("RGBA")
95
-
96
- # Check if image is fully transparent (no object)
97
- img_array = np.array(img)
98
- if np.all(img_array[:, :, 3] == 0):
99
- print(f"Warning: Image {image_path} is fully transparent or no object detected")
100
- return None
101
-
102
- # Create black background
103
- black_bg = Image.new("RGB", img.size, (0, 0, 0))
104
- black_bg.paste(img, (0, 0), img)
105
- return black_bg
106
- except Exception as e:
107
- print(f"Error in remove_background for {image_path}: {str(e)}")
108
- raise
 
 
 
 
 
 
109
 
110
- def preprocess_image(image_path):
111
- # Remove background and add black background
112
- img = remove_background(image_path)
113
- if img is None:
114
- raise ValueError("Image is fully transparent or no object detected")
115
-
116
- if img.width > MAX_DIMENSION or img.height > MAX_DIMENSION:
117
- if img.width > img.height:
118
- new_width = MAX_DIMENSION
119
- new_height = int(img.height * (MAX_DIMENSION / img.width))
120
- else:
121
- new_height = MAX_DIMENSION
122
- new_width = int(img.width * (MAX_DIMENSION / img.height))
123
- img = img.resize((new_width, new_height), Image.LANCZOS)
124
-
125
- img_array = np.array(img)
126
- if len(img_array.shape) == 3 and img_array.shape[2] == 3:
127
- lab = cv2.cvtColor(img_array, cv2.COLOR_RGB2LAB)
128
- l, a, b = cv2.split(lab)
129
- clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
130
- cl = clahe.apply(l)
131
- enhanced_lab = cv2.merge((cl, a, b))
132
- img_array = cv2.cvtColor(enhanced_lab, cv2.COLOR_LAB2RGB)
133
- img = Image.fromarray(img_array)
134
-
135
- return img
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
 
137
- def load_models():
138
- global dpt_estimator, depth_anything_model, depth_anything_processor, model_loaded, model_loading
139
-
140
- if model_loaded:
141
- return dpt_estimator, depth_anything_model, depth_anything_processor
142
-
143
- if model_loading:
144
- while model_loading and not model_loaded:
145
- time.sleep(0.5)
146
- return dpt_estimator, depth_anything_model, depth_anything_processor
147
-
148
- try:
149
- model_loading = True
150
- print("Loading models...")
151
-
152
- hf_token = os.environ.get('HF_TOKEN')
153
- if hf_token:
154
- print("HF_TOKEN found, attempting login...")
155
- login(token=hf_token)
156
- print("Authenticated with Hugging Face token")
157
- else:
158
- print("Warning: HF_TOKEN not found in environment")
159
-
160
- dpt_model_name = "Intel/dpt-large"
161
- max_retries = 3
162
- retry_delay = 5
163
- for attempt in range(max_retries):
164
- try:
165
- print(f"Attempting to download {dpt_model_name}, attempt {attempt+1}")
166
- snapshot_download(
167
- repo_id=dpt_model_name,
168
- cache_dir=CACHE_DIR,
169
- resume_download=True,
170
- token=hf_token
171
- )
172
- print(f"Successfully downloaded {dpt_model_name}")
173
- break
174
- except Exception as e:
175
- if attempt < max_retries - 1:
176
- print(f"DPT download attempt {attempt+1} failed: {str(e)}. Retrying after {retry_delay}s...")
177
- time.sleep(retry_delay)
178
- retry_delay *= 2
179
- else:
180
- raise
181
-
182
- dpt_estimator = pipeline(
183
- "depth-estimation",
184
- model=dpt_model_name,
185
- device=-1,
186
- cache_dir=CACHE_DIR,
187
- use_fast=True
188
- )
189
- print("DPT-Large loaded")
190
- gc.collect()
191
-
192
- da_model_name = "depth-anything/Depth-Anything-V2-Tiny-hf"
193
- for attempt in range(max_retries):
194
- try:
195
- print(f"Attempting to download {da_model_name}, attempt {attempt+1}")
196
- snapshot_download(
197
- repo_id=da_model_name,
198
- cache_dir=CACHE_DIR,
199
- resume_download=True,
200
- token=hf_token
201
- )
202
- print(f"Successfully downloaded {da_model_name}")
203
- break
204
- except Exception as e:
205
- if attempt < max_retries - 1:
206
- print(f"Depth Anything download attempt {attempt+1} failed: {str(e)}. Retrying after {retry_delay}s...")
207
- time.sleep(retry_delay)
208
- retry_delay *= 2
209
- else:
210
- print(f"Failed to load Depth Anything: {str(e)}. Falling back to DPT-Large only.")
211
- depth_anything_model = None
212
- depth_anything_processor = None
213
- model_loaded = True
214
- return dpt_estimator, None, None
215
-
216
- depth_anything_processor = AutoImageProcessor.from_pretrained(
217
- da_model_name,
218
- cache_dir=CACHE_DIR,
219
- token=hf_token
220
- )
221
- depth_anything_model = AutoModelForDepthEstimation.from_pretrained(
222
- da_model_name,
223
- cache_dir=CACHE_DIR,
224
- token=hf_token
225
- ).to("cpu")
226
-
227
- model_loaded = True
228
- print("Depth Anything loaded")
229
- return dpt_estimator, depth_anything_model, depth_anything_processor
230
-
231
- except Exception as e:
232
- print(f"Error loading models: {str(e)}")
233
- print(traceback.format_exc())
234
- raise
235
- finally:
236
- model_loading = False
237
 
238
- def fuse_depth_maps(dpt_depth, da_depth, detail_level='medium'):
239
- if isinstance(dpt_depth, Image.Image):
240
- dpt_depth = np.array(dpt_depth)
241
- if isinstance(da_depth, torch.Tensor):
242
- da_depth = da_depth.cpu().numpy()
243
- if len(dpt_depth.shape) > 2:
244
- dpt_depth = np.mean(dpt_depth, axis=2)
245
- if len(da_depth.shape) > 2:
246
- da_depth = np.mean(da_depth, axis=2)
247
-
248
- if dpt_depth.shape != da_depth.shape:
249
- da_depth = cv2.resize(da_depth, (dpt_depth.shape[1], dpt_depth.shape[0]), interpolation=cv2.INTER_CUBIC)
250
-
251
- p_low_dpt, p_high_dpt = np.percentile(dpt_depth, [1, 99])
252
- p_low_da, p_high_da = np.percentile(da_depth, [1, 99])
253
- dpt_depth = np.clip((dpt_depth - p_low_dpt) / (p_high_dpt - p_low_dpt), 0, 1) if p_high_dpt > p_low_dpt else dpt_depth
254
- da_depth = np.clip((da_depth - p_low_da) / (p_high_da - p_low_da), 0, 1) if p_high_da > p_low_da else da_depth
255
-
256
- if detail_level == 'high':
257
- weight_da = 0.7
258
- edges = cv2.Canny((da_depth * 255).astype(np.uint8), 50, 150)
259
- edge_mask = (edges > 0).astype(np.float32)
260
- dpt_weight = gaussian_filter(1 - edge_mask, sigma=1.0)
261
- da_weight = gaussian_filter(edge_mask, sigma=1.0)
262
- fused_depth = dpt_weight * dpt_depth + da_weight * da_depth * weight_da + (1 - weight_da) * dpt_depth
263
- else:
264
- weight_da = 0.5 if detail_level == 'medium' else 0.3
265
- fused_depth = (1 - weight_da) * dpt_depth + weight_da * da_depth
266
-
267
- fused_depth = np.clip(fused_depth, 0, 1)
268
- return fused_depth
269
 
270
- def enhance_depth_map(depth_map, detail_level='medium'):
271
- enhanced_depth = depth_map.copy().astype(np.float32)
272
- p_low, p_high = np.percentile(enhanced_depth, [1, 99])
273
- enhanced_depth = np.clip(enhanced_depth, p_low, p_high)
274
- enhanced_depth = (enhanced_depth - p_low) / (p_high - p_low) if p_high > p_low else enhanced_depth
275
-
276
- if detail_level == 'high':
277
- blurred = gaussian_filter(enhanced_depth, sigma=1.5)
278
- mask = enhanced_depth - blurred
279
- enhanced_depth = enhanced_depth + 1.5 * mask
280
- smooth1 = gaussian_filter(enhanced_depth, sigma=0.5)
281
- smooth2 = gaussian_filter(enhanced_depth, sigma=2.0)
282
- edge_mask = enhanced_depth - smooth2
283
- enhanced_depth = smooth1 + 1.2 * edge_mask
284
- elif detail_level == 'medium':
285
- blurred = gaussian_filter(enhanced_depth, sigma=1.0)
286
- mask = enhanced_depth - blurred
287
- enhanced_depth = enhanced_depth + 0.8 * mask
288
- enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.5)
289
- else:
290
- enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.7)
291
-
292
- enhanced_depth = np.clip(enhanced_depth, 0, 1)
293
- return enhanced_depth
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
294
 
295
- def depth_to_mesh(depth_map, image, resolution=80, detail_level='medium', view_angle=0):
296
- enhanced_depth = enhance_depth_map(depth_map, detail_level)
297
- h, w = enhanced_depth.shape
298
- x = np.linspace(0, w-1, resolution)
299
- y = np.linspace(0, h-1, resolution)
300
- x_grid, y_grid = np.meshgrid(x, y)
301
-
302
- interp_func = interpolate.RectBivariateSpline(
303
- np.arange(h), np.arange(w), enhanced_depth, kx=3, ky=3
304
- )
305
- z_values = interp_func(y, x, grid=True)
306
-
307
- if detail_level == 'high':
308
- dx = np.gradient(z_values, axis=1)
309
- dy = np.gradient(z_values, axis=0)
310
- gradient_magnitude = np.sqrt(dx**2 + dy**2)
311
- edge_mask = np.clip(gradient_magnitude * 5, 0, 0.2)
312
- z_values = z_values + edge_mask * (z_values - gaussian_filter(z_values, sigma=1.0))
313
-
314
- z_min, z_max = np.percentile(z_values, [2, 98])
315
- z_values = (z_values - z_min) / (z_max - z_min) if z_max > z_min else z_values
316
- z_scaling = 2.5 if detail_level == 'high' else 2.0 if detail_level == 'medium' else 1.5
317
- z_values = z_values * z_scaling
318
-
319
- x_grid = (x_grid / w - 0.5) * 2.0
320
- y_grid = (y_grid / h - 0.5) * 2.0
321
- vertices = np.vstack([x_grid.flatten(), -y_grid.flatten(), -z_values.flatten()]).T
322
-
323
- # Rotate vertices based on view angle (in radians)
324
- if view_angle != 0:
325
- rotation_matrix = trimesh.transformations.rotation_matrix(view_angle, [0, 1, 0])
326
- vertices = trimesh.transform_points(vertices, rotation_matrix)
327
-
328
- faces = []
329
- for i in range(resolution-1):
330
- for j in range(resolution-1):
331
- p1 = i * resolution + j
332
- p2 = i * resolution + (j + 1)
333
- p3 = (i + 1) * resolution + j
334
- p4 = (i + 1) * resolution + (j + 1)
335
- v1 = vertices[p1]
336
- v2 = vertices[p2]
337
- v3 = vertices[p3]
338
- v4 = vertices[p4]
339
- norm1 = np.cross(v2-v1, v4-v1)
340
- norm2 = np.cross(v4-v3, v1-v3)
341
- if np.dot(norm1, norm2) >= 0:
342
- faces.append([p1, p2, p4])
343
- faces.append([p1, p4, p3])
344
- else:
345
- faces.append([p1, p2, p3])
346
- faces.append([p2, p4, p3])
347
-
348
- faces = np.array(faces)
349
- mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
350
-
351
- if image:
352
- img_array = np.array(image)
353
- vertex_colors = np.zeros((vertices.shape[0], 4), dtype=np.uint8)
354
- for i in range(resolution):
355
- for j in range(resolution):
356
- img_x = j * (img_array.shape[1] - 1) / (resolution - 1)
357
- img_y = i * (img_array.shape[0] - 1) / (resolution - 1)
358
- x0, y0 = int(img_x), int(img_y)
359
- x1, y1 = min(x0 + 1, img_array.shape[1] - 1), min(y0 + 1, img_array.shape[0] - 1)
360
- wx = img_x - x0
361
- wy = img_y - y0
362
- vertex_idx = i * resolution + j
363
- if len(img_array.shape) == 3 and img_array.shape[2] == 3:
364
- r = int((1-wx)*(1-wy)*img_array[y0, x0, 0] + wx*(1-wy)*img_array[y0, x1, 0] +
365
- (1-wx)*wy*img_array[y1, x0, 0] + wx*wy*img_array[y1, x1, 0])
366
- g = int((1-wx)*(1-wy)*img_array[y0, x0, 1] + wx*(1-wy)*img_array[y0, x1, 1] +
367
- (1-wx)*wy*img_array[y1, x0, 1] + wx*wy*img_array[y1, x1, 1])
368
- b = int((1-wx)*(1-wy)*img_array[y0, x0, 2] + wx*(1-wy)*img_array[y0, x1, 2] +
369
- (1-wx)*wy*img_array[y1, x0, 2] + wx*wy*img_array[y1, x1, 2])
370
- vertex_colors[vertex_idx, :3] = [r, g, b]
371
- vertex_colors[vertex_idx, 3] = 255
372
- else:
373
- gray = int((1-wx)*(1-wy)*img_array[y0, x0] + wx*(1-wy)*img_array[y0, x1] +
374
- (1-wx)*wy*img_array[y1, x0] + wx*wy*img_array[y1, x1])
375
- vertex_colors[vertex_idx, :3] = [gray, gray, gray]
376
- vertex_colors[vertex_idx, 3] = 255
377
- mesh.visual.vertex_colors = vertex_colors
378
-
379
- if detail_level != 'high':
380
- mesh = mesh.smoothed(method='laplacian', iterations=1)
381
- mesh.fix_normals()
382
- return mesh
383
 
384
- def combine_meshes(meshes):
385
- if len(meshes) == 1:
386
- return meshes[0]
387
-
388
- combined_vertices = []
389
- combined_faces = []
390
- vertex_offset = 0
391
-
392
- for mesh in meshes:
393
- combined_vertices.append(mesh.vertices)
394
- combined_faces.append(mesh.faces + vertex_offset)
395
- vertex_offset += len(mesh.vertices)
396
-
397
- combined_vertices = np.vstack(combined_vertices)
398
- combined_faces = np.vstack(combined_faces)
399
-
400
- combined_mesh = trimesh.Trimesh(vertices=combined_vertices, faces=combined_faces)
401
-
402
- # Stitch overlapping vertices
403
- combined_mesh = combined_mesh.subdivide_to_size(max_edge=0.05)
404
- combined_mesh = combined_mesh.smoothed(method='laplacian', iterations=2)
405
-
406
- # Ensure watertight mesh
407
- combined_mesh.fill_holes()
408
- combined_mesh.fix_normals()
409
-
410
- return combined_mesh
411
 
412
- @app.route('/health', methods=['GET'])
413
- def health_check():
414
- return jsonify({
415
- "status": "healthy",
416
- "model": "DPT-Large + Depth Anything (Multi-View)",
417
- "device": "cpu"
418
- }), 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
419
 
420
- @app.route('/progress/<job_id>', methods=['GET'])
421
- def progress(job_id):
422
- def generate():
423
- if job_id not in processing_jobs:
424
- yield f"data: {json.dumps({'error': 'Job not found'})}\n\n"
425
- return
426
-
427
- job = processing_jobs[job_id]
428
- yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
429
-
430
- last_progress = job['progress']
431
- check_count = 0
432
- while job['status'] == 'processing':
433
- if job['progress'] != last_progress:
434
- yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
435
- last_progress = job['progress']
436
- time.sleep(0.5)
437
- check_count += 1
438
- if check_count > 60:
439
- if 'thread_alive' in job and not job['thread_alive']():
440
- job['status'] = 'error'
441
- job['error'] = 'Processing thread died unexpectedly'
442
- break
443
- check_count = 0
444
-
445
- if job['status'] == 'completed':
446
- yield f"data: {json.dumps({'status': 'completed', 'progress': 100, 'result_url': job['result_url'], 'preview_url': job['preview_url']})}\n\n"
447
- else:
448
- yield f"data: {json.dumps({'status': 'error', 'error': job['error']})}\n\n"
449
-
450
- return Response(stream_with_context(generate()), mimetype='text/event-stream')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
451
 
452
- @app.route('/convert', methods=['POST'])
453
- def convert_image_to_3d():
454
- required_views = ['front', 'back']
455
- optional_views = ['left', 'right']
456
- view_files = {}
457
-
458
- for view in required_views + optional_views:
459
- if view in request.files and request.files[view].filename != '':
460
- view_files[view] = request.files[view]
461
-
462
- if not all(view in view_files for view in required_views):
463
- return jsonify({"error": "Front and back images are required"}), 400
464
-
465
- for view, file in view_files.items():
466
- if not allowed_file(file.filename):
467
- return jsonify({"error": f"File type not allowed for {view}. Supported types: {', '.join(ALLOWED_EXTENSIONS)}"}), 400
468
-
469
- try:
470
- mesh_resolution = min(int(request.form.get('mesh_resolution', 80)), 120)
471
- output_format = request.form.get('output_format', 'glb').lower()
472
- detail_level = request.form.get('detail_level', 'medium').lower()
473
- texture_quality = request.form.get('texture_quality', 'medium').lower()
474
- except ValueError:
475
- return jsonify({"error": "Invalid parameter values"}), 400
476
-
477
- if output_format not in ['obj', 'glb']:
478
- return jsonify({"error": "Unsupported output format. Use 'obj' or 'glb'"}), 400
479
-
480
- if detail_level == 'high':
481
- mesh_resolution = min(int(mesh_resolution * 1.5), 120)
482
- elif detail_level == 'low':
483
- mesh_resolution = max(int(mesh_resolution * 0.7), 50)
484
-
485
- job_id = str(uuid.uuid4())
486
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
487
- os.makedirs(output_dir, exist_ok=True)
488
-
489
- filepaths = {}
490
- for view, file in view_files.items():
491
- filename = secure_filename(file.filename)
492
- filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{view}_{filename}")
493
- file.save(filepath)
494
- filepaths[view] = filepath
495
-
496
- processing_jobs[job_id] = {
497
- 'status': 'processing',
498
- 'progress': 0,
499
- 'result_url': None,
500
- 'preview_url': None,
501
- 'error': None,
502
- 'output_format': output_format,
503
- 'created_at': time.time()
504
- }
505
-
506
- def process_images():
507
- thread = threading.current_thread()
508
- processing_jobs[job_id]['thread_alive'] = lambda: thread.is_alive()
509
-
510
- try:
511
- processing_jobs[job_id]['progress'] = 5
512
- images = {}
513
- for view, filepath in filepaths.items():
514
- try:
515
- images[view] = preprocess_image(filepath)
516
- except ValueError as e:
517
- processing_jobs[job_id]['status'] = 'error'
518
- processing_jobs[job_id]['error'] = f"Error preprocessing {view} image: {str(e)}"
519
- return
520
- processing_jobs[job_id]['progress'] = 10
521
-
522
- try:
523
- dpt_model, da_model, da_processor = load_models()
524
- processing_jobs[job_id]['progress'] = 20
525
- except Exception as e:
526
- processing_jobs[job_id]['status'] = 'error'
527
- processing_jobs[job_id]['error'] = f"Error loading models: {str(e)}"
528
- return
529
-
530
- try:
531
- def estimate_depths():
532
- meshes = []
533
- view_angles = {'front': 0, 'back': np.pi, 'left': np.pi/2, 'right': -np.pi/2}
534
- with torch.no_grad():
535
- for view, image in images.items():
536
- # DPT-Large
537
- dpt_result = dpt_model(image)
538
- dpt_depth = dpt_result["depth"]
539
-
540
- # Depth Anything (if loaded)
541
- if da_model and da_processor:
542
- inputs = da_processor(images=image, return_tensors="pt")
543
- inputs = {k: v.to("cpu") for k, v in inputs.items()}
544
- outputs = da_model(**inputs)
545
- da_depth = outputs.predicted_depth.squeeze()
546
- da_depth = torch.nn.functional.interpolate(
547
- da_depth.unsqueeze(0).unsqueeze(0),
548
- size=(image.height, image.width),
549
- mode='bicubic',
550
- align_corners=False
551
- ).squeeze()
552
- fused_depth = fuse_depth_maps(dpt_depth, da_depth, detail_level)
553
- else:
554
- fused_depth = np.array(dpt_depth) if isinstance(dpt_depth, Image.Image) else dpt_depth
555
- if len(fused_depth.shape) > 2:
556
- fused_depth = np.mean(fused_depth, axis=2)
557
- p_low, p_high = np.percentile(fused_depth, [1, 99])
558
- fused_depth = np.clip((fused_depth - p_low) / (p_high - p_low), 0, 1) if p_high > p_low else fused_depth
559
-
560
- mesh = depth_to_mesh(fused_depth, image, resolution=mesh_resolution, detail_level=detail_level, view_angle=view_angles[view])
561
- meshes.append(mesh)
562
- gc.collect()
563
-
564
- combined_mesh = combine_meshes(meshes)
565
- return combined_mesh
566
-
567
- combined_mesh, error = process_with_timeout(estimate_depths, [], TIMEOUT_SECONDS)
568
-
569
- if error:
570
- if isinstance(error, TimeoutError):
571
- processing_jobs[job_id]['status'] = 'error'
572
- processing_jobs[job_id]['error'] = f"Processing timed out after {TIMEOUT_SECONDS} seconds"
573
- return
574
- else:
575
- raise error
576
-
577
- processing_jobs[job_id]['progress'] = 80
578
-
579
- if output_format == 'obj':
580
- obj_path = os.path.join(output_dir, "model.obj")
581
- combined_mesh.export(
582
- obj_path,
583
- file_type='obj',
584
- include_normals=True,
585
- include_texture=True
586
- )
587
- zip_path = os.path.join(output_dir, "model.zip")
588
- with zipfile.ZipFile(zip_path, 'w') as zipf:
589
- zipf.write(obj_path, arcname="model.obj")
590
- mtl_path = os.path.join(output_dir, "model.mtl")
591
- if os.path.exists(mtl_path):
592
- zipf.write(mtl_path, arcname="model.mtl")
593
- texture_path = os.path.join(output_dir, "model.png")
594
- if os.path.exists(texture_path):
595
- zipf.write(texture_path, arcname="model.png")
596
-
597
- processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
598
- processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
599
-
600
- elif output_format == 'glb':
601
- glb_path = os.path.join(output_dir, "model.glb")
602
- combined_mesh.export(
603
- glb_path,
604
- file_type='glb'
605
- )
606
- processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
607
- processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
608
-
609
- processing_jobs[job_id]['status'] = 'completed'
610
- processing_jobs[job_id]['progress'] = 100
611
- print(f"Job {job_id} completed")
612
-
613
- except Exception as e:
614
- error_details = traceback.format_exc()
615
- processing_jobs[job_id]['status'] = 'error'
616
- processing_jobs[job_id]['error'] = f"Error during processing: {str(e)}"
617
- print(f"Error processing job {job_id}: {str(e)}")
618
- print(error_details)
619
- return
620
-
621
- for filepath in filepaths.values():
622
- if os.path.exists(filepath):
623
- os.remove(filepath)
624
- gc.collect()
625
-
626
- except Exception as e:
627
- error_details = traceback.format_exc()
628
- processing_jobs[job_id]['status'] = 'error'
629
- processing_jobs[job_id]['error'] = f"{str(e)}\n{error_details}"
630
- print(f"Error processing job {job_id}: {str(e)}")
631
- print(error_details)
632
- for filepath in filepaths.values():
633
- if os.path.exists(filepath):
634
- os.remove(filepath)
635
-
636
- processing_thread = threading.Thread(target=process_images)
637
- processing_thread.daemon = True
638
- processing_thread.start()
639
-
640
- return jsonify({"job_id": job_id}), 202
641
 
642
- @app.route('/download/<job_id>', methods=['GET'])
643
- def download_model(job_id):
644
- if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
645
- return jsonify({"error": "Model not found or processing not complete"}), 404
646
-
647
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
648
- output_format = processing_jobs[job_id].get('output_format', 'glb')
649
-
650
- if output_format == 'obj':
651
- zip_path = os.path.join(output_dir, "model.zip")
652
- if os.path.exists(zip_path):
653
- return send_file(zip_path, as_attachment=True, download_name="model.zip")
654
- else:
655
- glb_path = os.path.join(output_dir, "model.glb")
656
- if os.path.exists(glb_path):
657
- return send_file(glb_path, as_attachment=True, download_name="model.glb")
658
-
659
- return jsonify({"error": "File not found"}), 404
660
 
661
- @app.route('/preview/<job_id>', methods=['GET'])
662
- def preview_model(job_id):
663
- if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
664
- return jsonify({"error": "Model not found or processing not complete"}), 404
665
-
666
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
667
- output_format = processing_jobs[job_id].get('output_format', 'glb')
668
-
669
- if output_format == 'obj':
670
- obj_path = os.path.join(output_dir, "model.obj")
671
- if os.path.exists(obj_path):
672
- return send_file(obj_path, mimetype='model/obj')
673
- else:
674
- glb_path = os.path.join(output_dir, "model.glb")
675
- if os.path.exists(glb_path):
676
- return send_file(glb_path, mimetype='model/gltf-binary')
677
-
678
- return jsonify({"error": "File not found"}), 404
 
 
 
 
 
679
 
680
- def cleanup_old_jobs():
681
- current_time = time.time()
682
- job_ids_to_remove = []
683
-
684
- for job_id, job_data in processing_jobs.items():
685
- if job_data['status'] == 'completed' and (current_time - job_data.get('created_at', 0)) > 3600:
686
- job_ids_to_remove.append(job_id)
687
- elif job_data['status'] == 'error' and (current_time - job_data.get('created_at', 0)) > 1800:
688
- job_ids_to_remove.append(job_id)
689
-
690
- for job_id in job_ids_to_remove:
691
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
692
- try:
693
- import shutil
694
- if os.path.exists(output_dir):
695
- shutil.rmtree(output_dir)
696
- except Exception as e:
697
- print(f"Error cleaning up job {job_id}: {str(e)}")
698
-
699
- if job_id in processing_jobs:
700
- del processing_jobs[job_id]
701
-
702
- threading.Timer(300, cleanup_old_jobs).start()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
703
 
704
- @app.route('/model-info/<job_id>', methods=['GET'])
705
- def model_info(job_id):
706
- if job_id not in processing_jobs:
707
- return jsonify({"error": "Model not found"}), 404
708
-
709
- job = processing_jobs[job_id]
710
-
711
- if job['status'] != 'completed':
712
- return jsonify({
713
- "status": job['status'],
714
- "progress": job['progress'],
715
- "error": job.get('error')
716
- }), 200
717
-
718
- output_dir = os.path.join(RESULTS_FOLDER, job_id)
719
- model_stats = {}
720
-
721
- if job['output_format'] == 'obj':
722
- obj_path = os.path.join(output_dir, "model.obj")
723
- zip_path = os.path.join(output_dir, "model.zip")
724
- if os.path.exists(obj_path):
725
- model_stats['obj_size'] = os.path.getsize(obj_path)
726
- if os.path.exists(zip_path):
727
- model_stats['package_size'] = os.path.getsize(zip_path)
728
- else:
729
- glb_path = os.path.join(output_dir, "model.glb")
730
- if os.path.exists(glb_path):
731
- model_stats['model_size'] = os.path.getsize(glb_path)
732
-
733
- return jsonify({
734
- "status": job['status'],
735
- "model_format": job['output_format'],
736
- "download_url": job['result_url'],
737
- "preview_url": job['preview_url'],
738
- "model_stats": model_stats,
739
- "created_at": job.get('created_at'),
740
- "completed_at": job.get('completed_at')
741
- }), 200
742
 
743
- @app.route('/', methods=['GET'])
744
- def index():
745
- return jsonify({
746
- "message": "Multi-View Image to 3D API (DPT-Large + Depth Anything)",
747
- "endpoints": [
748
- "/convert",
749
- "/progress/<job_id>",
750
- "/download/<job_id>",
751
- "/preview/<job_id>",
752
- "/model-info/<job_id>"
753
- ],
754
- "parameters": {
755
- "front": "Image file (required)",
756
- "back": "Image file (required)",
757
- "left": "Image file (optional)",
758
- "right": "Image file (optional)",
759
- "mesh_resolution": "Integer (50-120)",
760
- "output_format": "obj or glb",
761
- "detail_level": "low, medium, or high",
762
- "texture_quality": "low, medium, or high"
763
- },
764
- "description": "Creates high-quality 3D models from multiple 2D images (front, back, left, right) using DPT-Large and Depth Anything."
765
- }), 200
766
-
767
- if __name__ == '__main__':
768
- cleanup_old_jobs()
769
- port = int(os.environ.get('PORT', 7860))
770
- app.run(host='0.0.0.0', port=port)
771
-
 
1
+ import os
2
+ import torch
3
+ import time
4
+ import threading
5
+ import json
6
+ import gc
7
+ from flask import Flask, request, jsonify, send_file, Response, stream_with_context
8
+ from werkzeug.utils import secure_filename
9
+ from PIL import Image
10
+ import io
11
+ import zipfile
12
+ import uuid
13
+ import traceback
14
+ from huggingface_hub import snapshot_download, login
15
+ from flask_cors import CORS
16
+ import numpy as np
17
+ import trimesh
18
+ from transformers import pipeline, AutoImageProcessor, AutoModelForDepthEstimation
19
+ from scipy.ndimage import gaussian_filter
20
+ from scipy import interpolate
21
+ import cv2
22
+ from rembg import remove
23
 
24
+ app = Flask(__name__)
25
+ CORS(app)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
 
27
+ # Configure directories
28
+ UPLOAD_FOLDER = '/tmp/uploads'
29
+ RESULTS_FOLDER = '/tmp/results'
30
+ CACHE_DIR = '/tmp/huggingface'
31
+ ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg'}
32
 
33
+ os.makedirs(UPLOAD_FOLDER, exist_ok=True)
34
+ os.makedirs(RESULTS_FOLDER, exist_ok=True)
35
+ os.makedirs(CACHE_DIR, exist_ok=True)
 
 
36
 
37
+ os.environ['HF_HOME'] = CACHE_DIR
38
+ app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
39
+ app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
40
 
41
+ # Job tracking
42
+ processing_jobs = {}
 
43
 
44
+ # Model variables
45
+ dpt_estimator = None
46
+ depth_anything_model = None
47
+ depth_anything_processor = None
48
+ model_loaded = False
49
+ model_loading = False
50
 
51
+ TIMEOUT_SECONDS = 240
52
+ MAX_DIMENSION = 518
 
 
 
 
53
 
54
+ class TimeoutError(Exception):
55
+ pass
56
 
57
+ def process_with_timeout(function, args, timeout):
58
+ result = [None]
59
+ error = [None]
60
+ completed = [False]
61
+
62
+ def target():
63
+ try:
64
+ result[0] = function(*args)
65
+ completed[0] = True
66
+ except Exception as e:
67
+ error[0] = e
68
+
69
+ thread = threading.Thread(target=target)
70
+ thread.daemon = True
71
+ thread.start()
72
+ thread.join(timeout)
73
+
74
+ if not completed[0]:
75
+ if thread.is_alive():
76
+ return None, TimeoutError(f"Processing timed out after {timeout} seconds")
77
+ elif error[0]:
78
+ return None, error[0]
79
+
80
+ if error[0]:
81
+ return None, error[0]
82
+
83
+ return result[0], None
84
 
85
+ def allowed_file(filename):
86
+ return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
 
88
+ def remove_background(image_path):
89
+ try:
90
+ with open(image_path, "rb") as img_file:
91
+ img_data = img_file.read()
92
+ result = remove(img_data)
93
+ img = Image.open(io.BytesIO(result)).convert("RGBA")
94
+
95
+ # Check if image is fully transparent (no object)
96
+ img_array = np.array(img)
97
+ if np.all(img_array[:, :, 3] == 0):
98
+ print(f"Warning: Image {image_path} is fully transparent or no object detected")
99
+ return None
100
+
101
+ # Create black background
102
+ black_bg = Image.new("RGB", img.size, (0, 0, 0))
103
+ black_bg.paste(img, (0, 0), img)
104
+ return black_bg
105
+ except Exception as e:
106
+ print(f"Error in remove_background for {image_path}: {str(e)}")
107
+ raise
108
 
109
+ def preprocess_image(image_path):
110
+ # Remove background and add black background
111
+ img = remove_background(image_path)
112
+ if img is None:
113
+ raise ValueError("Image is fully transparent or no object detected")
114
+
115
+ if img.width > MAX_DIMENSION or img.height > MAX_DIMENSION:
116
+ if img.width > img.height:
117
+ new_width = MAX_DIMENSION
118
+ new_height = int(img.height * (MAX_DIMENSION / img.width))
119
+ else:
120
+ new_height = MAX_DIMENSION
121
+ new_width = int(img.width * (MAX_DIMENSION / img.height))
122
+ img = img.resize((new_width, new_height), Image.LANCZOS)
123
+
124
+ img_array = np.array(img)
125
+ if len(img_array.shape) == 3 and img_array.shape[2] == 3:
126
+ lab = cv2.cvtColor(img_array, cv2.COLOR_RGB2LAB)
127
+ l, a, b = cv2.split(lab)
128
+ clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8))
129
+ cl = clahe.apply(l)
130
+ enhanced_lab = cv2.merge((cl, a, b))
131
+ img_array = cv2.cvtColor(enhanced_lab, cv2.COLOR_LAB2RGB)
132
+ img = Image.fromarray(img_array)
133
+
134
+ return img
135
 
136
+ def load_models():
137
+ global dpt_estimator, depth_anything_model, depth_anything_processor, model_loaded, model_loading
138
+
139
+ if model_loaded:
140
+ return dpt_estimator, depth_anything_model, depth_anything_processor
141
+
142
+ if model_loading:
143
+ while model_loading and not model_loaded:
144
+ time.sleep(0.5)
145
+ return dpt_estimator, depth_anything_model, depth_anything_processor
146
+
147
+ try:
148
+ model_loading = True
149
+ print("Loading models...")
150
+
151
+ hf_token = os.environ.get('HF_TOKEN')
152
+ if hf_token:
153
+ print("HF_TOKEN found, attempting login...")
154
+ login(token=hf_token)
155
+ print("Authenticated with Hugging Face token")
156
+ else:
157
+ print("Warning: HF_TOKEN not found in environment")
158
+
159
+ dpt_model_name = "Intel/dpt-large"
160
+ max_retries = 3
161
+ retry_delay = 5
162
+ for attempt in range(max_retries):
163
+ try:
164
+ print(f"Attempting to download {dpt_model_name}, attempt {attempt+1}")
165
+ snapshot_download(
166
+ repo_id=dpt_model_name,
167
+ cache_dir=CACHE_DIR,
168
+ resume_download=True,
169
+ token=hf_token
170
+ )
171
+ print(f"Successfully downloaded {dpt_model_name}")
172
+ break
173
+ except Exception as e:
174
+ if attempt < max_retries - 1:
175
+ print(f"DPT download attempt {attempt+1} failed: {str(e)}. Retrying after {retry_delay}s...")
176
+ time.sleep(retry_delay)
177
+ retry_delay *= 2
178
+ else:
179
+ raise
180
+
181
+ dpt_estimator = pipeline(
182
+ "depth-estimation",
183
+ model=dpt_model_name,
184
+ device=-1,
185
+ cache_dir=CACHE_DIR,
186
+ use_fast=True
187
+ )
188
+ print("DPT-Large loaded")
189
+ gc.collect()
190
+
191
+ da_model_name = "depth-anything/Depth-Anything-V2-Tiny-hf"
192
+ for attempt in range(max_retries):
193
+ try:
194
+ print(f"Attempting to download {da_model_name}, attempt {attempt+1}")
195
+ snapshot_download(
196
+ repo_id=da_model_name,
197
+ cache_dir=CACHE_DIR,
198
+ resume_download=True,
199
+ token=hf_token
200
+ )
201
+ print(f"Successfully downloaded {da_model_name}")
202
+ break
203
+ except Exception as e:
204
+ if attempt < max_retries - 1:
205
+ print(f"Depth Anything download attempt {attempt+1} failed: {str(e)}. Retrying after {retry_delay}s...")
206
+ time.sleep(retry_delay)
207
+ retry_delay *= 2
208
+ else:
209
+ print(f"Failed to load Depth Anything: {str(e)}. Falling back to DPT-Large only.")
210
+ depth_anything_model = None
211
+ depth_anything_processor = None
212
+ model_loaded = True
213
+ return dpt_estimator, None, None
214
+
215
+ depth_anything_processor = AutoImageProcessor.from_pretrained(
216
+ da_model_name,
217
+ cache_dir=CACHE_DIR,
218
+ token=hf_token
219
+ )
220
+ depth_anything_model = AutoModelForDepthEstimation.from_pretrained(
221
+ da_model_name,
222
+ cache_dir=CACHE_DIR,
223
+ token=hf_token
224
+ ).to("cpu")
225
+
226
+ model_loaded = True
227
+ print("Depth Anything loaded")
228
+ return dpt_estimator, depth_anything_model, depth_anything_processor
229
+
230
+ except Exception as e:
231
+ print(f"Error loading models: {str(e)}")
232
+ print(traceback.format_exc())
233
+ raise
234
+ finally:
235
+ model_loading = False
236
 
237
+ def fuse_depth_maps(dpt_depth, da_depth, detail_level='medium'):
238
+ if isinstance(dpt_depth, Image.Image):
239
+ dpt_depth = np.array(dpt_depth)
240
+ if isinstance(da_depth, torch.Tensor):
241
+ da_depth = da_depth.cpu().numpy()
242
+ if len(dpt_depth.shape) > 2:
243
+ dpt_depth = np.mean(dpt_depth, axis=2)
244
+ if len(da_depth.shape) > 2:
245
+ da_depth = np.mean(da_depth, axis=2)
246
+
247
+ if dpt_depth.shape != da_depth.shape:
248
+ da_depth = cv2.resize(da_depth, (dpt_depth.shape[1], dpt_depth.shape[0]), interpolation=cv2.INTER_CUBIC)
249
+
250
+ p_low_dpt, p_high_dpt = np.percentile(dpt_depth, [1, 99])
251
+ p_low_da, p_high_da = np.percentile(da_depth, [1, 99])
252
+ dpt_depth = np.clip((dpt_depth - p_low_dpt) / (p_high_dpt - p_low_dpt), 0, 1) if p_high_dpt > p_low_dpt else dpt_depth
253
+ da_depth = np.clip((da_depth - p_low_da) / (p_high_da - p_low_da), 0, 1) if p_high_da > p_low_da else da_depth
254
+
255
+ if detail_level == 'high':
256
+ weight_da = 0.7
257
+ edges = cv2.Canny((da_depth * 255).astype(np.uint8), 50, 150)
258
+ edge_mask = (edges > 0).astype(np.float32)
259
+ dpt_weight = gaussian_filter(1 - edge_mask, sigma=1.0)
260
+ da_weight = gaussian_filter(edge_mask, sigma=1.0)
261
+ fused_depth = dpt_weight * dpt_depth + da_weight * da_depth * weight_da + (1 - weight_da) * dpt_depth
262
+ else:
263
+ weight_da = 0.5 if detail_level == 'medium' else 0.3
264
+ fused_depth = (1 - weight_da) * dpt_depth + weight_da * da_depth
265
+
266
+ fused_depth = np.clip(fused_depth, 0, 1)
267
+ return fused_depth
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
268
 
269
+ def enhance_depth_map(depth_map, detail_level='medium'):
270
+ enhanced_depth = depth_map.copy().astype(np.float32)
271
+ p_low, p_high = np.percentile(enhanced_depth, [1, 99])
272
+ enhanced_depth = np.clip(enhanced_depth, p_low, p_high)
273
+ enhanced_depth = (enhanced_depth - p_low) / (p_high - p_low) if p_high > p_low else enhanced_depth
274
+
275
+ if detail_level == 'high':
276
+ blurred = gaussian_filter(enhanced_depth, sigma=1.5)
277
+ mask = enhanced_depth - blurred
278
+ enhanced_depth = enhanced_depth + 1.5 * mask
279
+ smooth1 = gaussian_filter(enhanced_depth, sigma=0.5)
280
+ smooth2 = gaussian_filter(enhanced_depth, sigma=2.0)
281
+ edge_mask = enhanced_depth - smooth2
282
+ enhanced_depth = smooth1 + 1.2 * edge_mask
283
+ elif detail_level == 'medium':
284
+ blurred = gaussian_filter(enhanced_depth, sigma=1.0)
285
+ mask = enhanced_depth - blurred
286
+ enhanced_depth = enhanced_depth + 0.8 * mask
287
+ enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.5)
288
+ else:
289
+ enhanced_depth = gaussian_filter(enhanced_depth, sigma=0.7)
290
+
291
+ fused_depth = np.clip(fused_depth, 0, 1)
292
+ return fused_depth
 
 
 
 
 
 
 
293
 
294
+ def depth_to_mesh(depth_map, image, resolution=80, detail_level='medium', view_angle=0):
295
+ enhanced_depth = enhance_depth_map(depth_map, detail_level)
296
+ h, w = enhanced_depth.shape
297
+ x = np.linspace(0, w-1, resolution)
298
+ y = np.linspace(0, h-1, resolution)
299
+ x_grid, y_grid = np.meshgrid(x, y)
300
+
301
+ interp_func = interpolate.RectBivariateSpline(
302
+ np.arange(h), np.arange(w), enhanced_depth, kx=3, ky=3
303
+ )
304
+ z_values = interp_func(y, x, grid=True)
305
+
306
+ if detail_level == 'high':
307
+ dx = np.gradient(z_values, axis=1)
308
+ dy = np.gradient(z_values, axis=0)
309
+ gradient_magnitude = np.sqrt(dx**2 + dy**2)
310
+ edge_mask = np.clip(gradient_magnitude * 5, 0, 0.2)
311
+ z_values = z_values + edge_mask * (z_values - gaussian_filter(z_values, sigma=1.0))
312
+
313
+ z_min, z_max = np.percentile(z_values, [2, 98])
314
+ z_values = (z_values - z_min) / (z_max - z_min) if z_max > z_min else z_values
315
+ z_scaling = 2.5 if detail_level == 'high' else 2.0 if detail_level == 'medium' else 1.5
316
+ z_values = z_values * z_scaling
317
+
318
+ x_grid = (x_grid / w - 0.5) * 2.0
319
+ y_grid = (y_grid / h - 0.5) * 2.0
320
+ vertices = np.vstack([x_grid.flatten(), -y_grid.flatten(), -z_values.flatten()]).T
321
+
322
+ # Rotate vertices based on view angle (in radians)
323
+ if view_angle != 0:
324
+ rotation_matrix = trimesh.transformations.rotation_matrix(view_angle, [0, 1, 0])
325
+ vertices = trimesh.transform_points(vertices, rotation_matrix)
326
+
327
+ faces = []
328
+ for i in range(resolution-1):
329
+ for j in range(resolution-1):
330
+ p1 = i * resolution + j
331
+ p2 = i * resolution + (j + 1)
332
+ p3 = (i + 1) * resolution + j
333
+ p4 = (i + 1) * resolution + (j + 1)
334
+ v1 = vertices[p1]
335
+ v2 = vertices[p2]
336
+ v3 = vertices[p3]
337
+ v4 = vertices[p4]
338
+ norm1 = np.cross(v2-v1, v4-v1)
339
+ norm2 = np.cross(v4-v3, v1-v3)
340
+ if np.dot(norm1, norm2) >= 0:
341
+ faces.append([p1, p2, p4])
342
+ faces.append([p1, p4, p3])
343
+ else:
344
+ faces.append([p1, p2, p3])
345
+ faces.append([p2, p4, p3])
346
+
347
+ faces = np.array(faces)
348
+ mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
349
+
350
+ if image:
351
+ img_array = np.array(image)
352
+ vertex_colors = np.zeros((vertices.shape[0], 4), dtype=np.uint8)
353
+ for i in range(resolution):
354
+ for j in range(resolution):
355
+ img_x = j * (img_array.shape[1] - 1) / (resolution - 1)
356
+ img_y = i * (img_array.shape[0] - 1) / (resolution - 1)
357
+ x0, y0 = int(img_x), int(img_y)
358
+ x1, y1 = min(x0 + 1, img_array.shape[1] - 1), min(y0 + 1, img_array.shape[0] - 1)
359
+ wx = img_x - x0
360
+ wy = img_y - y0
361
+ vertex_idx = i * resolution + j
362
+ if len(img_array.shape) == 3 and img_array.shape[2] == 3:
363
+ r = int((1-wx)*(1-wy)*img_array[y0, x0, 0] + wx*(1-wy)*img_array[y0, x1, 0] +
364
+ (1-wx)*wy*img_array[y1, x0, 0] + wx*wy*img_array[y1, x1, 0])
365
+ g = int((1-wx)*(1-wy)*img_array[y0, x0, 1] + wx*(1-wy)*img_array[y0, x1, 1] +
366
+ (1-wx)*wy*img_array[y1, x0, 1] + wx*wy*img_array[y1, x1, 1])
367
+ b = int((1-wx)*(1-wy)*img_array[y0, x0, 2] + wx*(1-wy)*img_array[y0, x1, 2] +
368
+ (1-wx)*wy*img_array[y1, x0, 2] + wx*wy*img_array[y1, x1, 2])
369
+ vertex_colors[vertex_idx, :3] = [r, g, b]
370
+ vertex_colors[vertex_idx, 3] = 255
371
+ else:
372
+ gray = int((1-wx)*(1-wy)*img_array[y0, x0] + wx*(1-wy)*img_array[y0, x1] +
373
+ (1-wx)*wy*img_array[y1, x0] + wx*wy*img_array[y1, x1])
374
+ vertex_colors[vertex_idx, :3] = [gray, gray, gray]
375
+ vertex_colors[vertex_idx, 3] = 255
376
+ mesh.visual.vertex_colors = vertex_colors
377
+
378
+ if detail_level != 'high':
379
+ mesh = mesh.smoothed(method='laplacian', iterations=1)
380
+ mesh.fix_normals()
381
+ return mesh
382
 
383
+ def combine_meshes(meshes):
384
+ if len(meshes) == 1:
385
+ return meshes[0]
386
+
387
+ combined_vertices = []
388
+ combined_faces = []
389
+ vertex_offset = 0
390
+
391
+ for mesh in meshes:
392
+ combined_vertices.append(mesh.vertices)
393
+ combined_faces.append(mesh.faces + vertex_offset)
394
+ vertex_offset += len(mesh.vertices)
395
+
396
+ combined_vertices = np.vstack(combined_vertices)
397
+ combined_faces = np.vstack(combined_faces)
398
+
399
+ combined_mesh = trimesh.Trimesh(vertices=combined_vertices, faces=combined_faces)
400
+
401
+ # Stitch overlapping vertices
402
+ combined_mesh = combined_mesh.subdivide_to_size(max_edge=0.05)
403
+ combined_mesh = combined_mesh.smoothed(method='laplacian', iterations=2)
404
+
405
+ # Ensure watertight mesh
406
+ combined_mesh.fill_holes()
407
+ combined_mesh.fix_normals()
408
+
409
+ return combined_mesh
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
410
 
411
+ @app.route('/health', methods=['GET'])
412
+ def health_check():
413
+ return jsonify({
414
+ "status": "healthy",
415
+ "model": "DPT-Large + Depth Anything (Multi-View)",
416
+ "device": "cpu"
417
+ }), 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
418
 
419
+ @app.route('/progress/<job_id>', methods=['GET'])
420
+ def progress(job_id):
421
+ def generate():
422
+ if job_id not in processing_jobs:
423
+ yield f"data: {json.dumps({'error': 'Job not found'})}\n\n"
424
+ return
425
+
426
+ job = processing_jobs[job_id]
427
+ yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
428
+
429
+ last_progress = job['progress']
430
+ check_count = 0
431
+ while job['status'] == 'processing':
432
+ if job['progress'] != last_progress:
433
+ yield f"data: {json.dumps({'status': 'processing', 'progress': job['progress']})}\n\n"
434
+ last_progress = job['progress']
435
+ time.sleep(0.5)
436
+ check_count += 1
437
+ if check_count > 60:
438
+ if 'thread_alive' in job and not job['thread_alive']():
439
+ job['status'] = 'error'
440
+ job['error'] = 'Processing thread died unexpectedly'
441
+ break
442
+ check_count = 0
443
+
444
+ if job['status'] == 'completed':
445
+ yield f"data: {json.dumps({'status': 'completed', 'progress': 100, 'result_url': job['result_url'], 'preview_url': job['preview_url']})}\n\n"
446
+ else:
447
+ yield f"data: {json.dumps({'status': 'error', 'error': job['error']})}\n\n"
448
+
449
+ return Response(stream_with_context(generate()), mimetype='text/event-stream')
450
 
451
+ @app.route('/convert', methods=['POST'])
452
+ def convert_image_to_3d():
453
+ required_views = ['front', 'back']
454
+ optional_views = ['left', 'right']
455
+ view_files = {}
456
+
457
+ for view in required_views + optional_views:
458
+ if view in request.files and request.files[view].filename != '':
459
+ view_files[view] = request.files[view]
460
+
461
+ if not all(view in view_files for view in required_views):
462
+ return jsonify({"error": "Front and back images are required"}), 400
463
+
464
+ for view, file in view_files.items():
465
+ if not allowed_file(file.filename):
466
+ return jsonify({"error": f"File type not allowed for {view}. Supported types: {', '.join(ALLOWED_EXTENSIONS)}"}), 400
467
+
468
+ try:
469
+ mesh_resolution = min(int(request.form.get('mesh_resolution', 80)), 120)
470
+ output_format = request.form.get('output_format', 'glb').lower()
471
+ detail_level = request.form.get('detail_level', 'medium').lower()
472
+ texture_quality = request.form.get('texture_quality', 'medium').lower()
473
+ except ValueError:
474
+ return jsonify({"error": "Invalid parameter values"}), 400
475
+
476
+ if output_format not in ['obj', 'glb']:
477
+ return jsonify({"error": "Unsupported output format. Use 'obj' or 'glb'"}), 400
478
+
479
+ if detail_level == 'high':
480
+ mesh_resolution = min(int(mesh_resolution * 1.5), 120)
481
+ elif detail_level == 'low':
482
+ mesh_resolution = max(int(mesh_resolution * 0.7), 50)
483
+
484
+ job_id = str(uuid.uuid4())
485
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
486
+ os.makedirs(output_dir, exist_ok=True)
487
+
488
+ filepaths = {}
489
+ for view, file in view_files.items():
490
+ filename = secure_filename(file.filename)
491
+ filepath = os.path.join(app.config['UPLOAD_FOLDER'], f"{job_id}_{view}_{filename}")
492
+ file.save(filepath)
493
+ filepaths[view] = filepath
494
+
495
+ processing_jobs[job_id] = {
496
+ 'status': 'processing',
497
+ 'progress': 0,
498
+ 'result_url': None,
499
+ 'preview_url': None,
500
+ 'error': None,
501
+ 'output_format': output_format,
502
+ 'created_at': time.time()
503
+ }
504
+
505
+ def process_images():
506
+ thread = threading.current_thread()
507
+ processing_jobs[job_id]['thread_alive'] = lambda: thread.is_alive()
508
+
509
+ try:
510
+ processing_jobs[job_id]['progress'] = 5
511
+ images = {}
512
+ for view, filepath in filepaths.items():
513
+ try:
514
+ images[view] = preprocess_image(filepath)
515
+ except ValueError as e:
516
+ processing_jobs[job_id]['status'] = 'error'
517
+ processing_jobs[job_id]['error'] = f"Error preprocessing {view} image: {str(e)}"
518
+ return
519
+ processing_jobs[job_id]['progress'] = 10
520
+
521
+ try:
522
+ dpt_model, da_model, da_processor = load_models()
523
+ processing_jobs[job_id]['progress'] = 20
524
+ except Exception as e:
525
+ processing_jobs[job_id]['status'] = 'error'
526
+ processing_jobs[job_id]['error'] = f"Error loading models: {str(e)}"
527
+ return
528
+
529
+ try:
530
+ def estimate_depths():
531
+ meshes = []
532
+ view_angles = {'front': 0, 'back': np.pi, 'left': np.pi/2, 'right': -np.pi/2}
533
+ with torch.no_grad():
534
+ for view, image in images.items():
535
+ # DPT-Large
536
+ dpt_result = dpt_model(image)
537
+ dpt_depth = dpt_result["depth"]
538
+
539
+ # Depth Anything (if loaded)
540
+ if da_model and da_processor:
541
+ inputs = da_processor(images=image, return_tensors="pt")
542
+ inputs = {k: v.to("cpu") for k, v in inputs.items()}
543
+ outputs = da_model(**inputs)
544
+ da_depth = outputs.predicted_depth.squeeze()
545
+ da_depth = torch.nn.functional.interpolate(
546
+ da_depth.unsqueeze(0).unsqueeze(0),
547
+ size=(image.height, image.width),
548
+ mode='bicubic',
549
+ align_corners=False
550
+ ).squeeze()
551
+ fused_depth = fuse_depth_maps(dpt_depth, da_depth, detail_level)
552
+ else:
553
+ fused_depth = np.array(dpt_depth) if isinstance(dpt_depth, Image.Image) else dpt_depth
554
+ if len(fused_depth.shape) > 2:
555
+ fused_depth = np.mean(fused_depth, axis=2)
556
+ p_low, p_high = np.percentile(fused_depth, [1, 99])
557
+ fused_depth = np.clip((fused_depth - p_low) / (p_high - p_low), 0, 1) if p_high > p_low else fused_depth
558
+
559
+ mesh = depth_to_mesh(fused_depth, image, resolution=mesh_resolution, detail_level=detail_level, view_angle=view_angles[view])
560
+ meshes.append(mesh)
561
+ gc.collect()
562
+
563
+ combined_mesh = combine_meshes(meshes)
564
+ return combined_mesh
565
+
566
+ combined_mesh, error = process_with_timeout(estimate_depths, [], TIMEOUT_SECONDS)
567
+
568
+ if error:
569
+ if isinstance(error, TimeoutError):
570
+ processing_jobs[job_id]['status'] = 'error'
571
+ processing_jobs[job_id]['error'] = f"Processing timed out after {TIMEOUT_SECONDS} seconds"
572
+ return
573
+ else:
574
+ raise error
575
+
576
+ processing_jobs[job_id]['progress'] = 80
577
+
578
+ if output_format == 'obj':
579
+ obj_path = os.path.join(output_dir, "model.obj")
580
+ combined_mesh.export(
581
+ obj_path,
582
+ file_type='obj',
583
+ include_normals=True,
584
+ include_texture=True
585
+ )
586
+ zip_path = os.path.join(output_dir, "model.zip")
587
+ with zipfile.ZipFile(zip_path, 'w') as zipf:
588
+ zipf.write(obj_path, arcname="model.obj")
589
+ mtl_path = os.path.join(output_dir, "model.mtl")
590
+ if os.path.exists(mtl_path):
591
+ zipf.write(mtl_path, arcname="model.mtl")
592
+ texture_path = os.path.join(output_dir, "model.png")
593
+ if os.path.exists(texture_path):
594
+ zipf.write(texture_path, arcname="model.png")
595
+
596
+ processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
597
+ processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
598
+
599
+ elif output_format == 'glb':
600
+ glb_path = os.path.join(output_dir, "model.glb")
601
+ combined_mesh.export(
602
+ glb_path,
603
+ file_type='glb'
604
+ )
605
+ processing_jobs[job_id]['result_url'] = f"/download/{job_id}"
606
+ processing_jobs[job_id]['preview_url'] = f"/preview/{job_id}"
607
+
608
+ processing_jobs[job_id]['status'] = 'completed'
609
+ processing_jobs[job_id]['progress'] = 100
610
+ print(f"Job {job_id} completed")
611
+
612
+ except Exception as e:
613
+ error_details = traceback.format_exc()
614
+ processing_jobs[job_id]['status'] = 'error'
615
+ processing_jobs[job_id]['error'] = f"Error during processing: {str(e)}"
616
+ print(f"Error processing job {job_id}: {str(e)}")
617
+ print(error_details)
618
+ return
619
+
620
+ for filepath in filepaths.values():
621
+ if os.path.exists(filepath):
622
+ os.remove(filepath)
623
+ gc.collect()
624
+
625
+ except Exception as e:
626
+ error_details = traceback.format_exc()
627
+ processing_jobs[job_id]['status'] = 'error'
628
+ processing_jobs[job_id]['error'] = f"{str(e)}\n{error_details}"
629
+ print(f"Error processing job {job_id}: {str(e)}")
630
+ print(error_details)
631
+ for filepath in filepaths.values():
632
+ if os.path.exists(filepath):
633
+ os.remove(filepath)
634
+
635
+ processing_thread = threading.Thread(target=process_images)
636
+ processing_thread.daemon = True
637
+ processing_thread.start()
638
+
639
+ return jsonify({"job_id": job_id}), 202
640
 
641
+ @app.route('/download/<job_id>', methods=['GET'])
642
+ def download_model(job_id):
643
+ if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
644
+ return jsonify({"error": "Model not found or processing not complete"}), 404
645
+
646
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
647
+ output_format = processing_jobs[job_id].get('output_format', 'glb')
648
+
649
+ if output_format == 'obj':
650
+ zip_path = os.path.join(output_dir, "model.zip")
651
+ if os.path.exists(zip_path):
652
+ return send_file(zip_path, as_attachment=True, download_name="model.zip")
653
+ else:
654
+ glb_path = os.path.join(output_dir, "model.glb")
655
+ if os.path.exists(glb_path):
656
+ return send_file(glb_path, as_attachment=True, download_name="model.glb")
657
+
658
+ return jsonify({"error": "File not found"}), 404
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
659
 
660
+ @app.route('/preview/<job_id>', methods=['GET'])
661
+ def preview_model(job_id):
662
+ if job_id not in processing_jobs or processing_jobs[job_id]['status'] != 'completed':
663
+ return jsonify({"error": "Model not found or processing not complete"}), 404
664
+
665
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
666
+ output_format = processing_jobs[job_id].get('output_format', 'glb')
667
+
668
+ if output_format == 'obj':
669
+ obj_path = os.path.join(output_dir, "model.obj")
670
+ if os.path.exists(obj_path):
671
+ return send_file(obj_path, mimetype='model/obj')
672
+ else:
673
+ glb_path = os.path.join(output_dir, "model.glb")
674
+ if os.path.exists(glb_path):
675
+ return send_file(glb_path, mimetype='model/gltf-binary')
676
+
677
+ return jsonify({"error": "File not found"}), 404
678
 
679
+ def cleanup_old_jobs():
680
+ current_time = time.time()
681
+ job_ids_to_remove = []
682
+
683
+ for job_id, job_data in processing_jobs.items():
684
+ if job_data['status'] == 'completed' and (current_time - job_data.get('created_at', 0)) > 3600:
685
+ job_ids_to_remove.append(job_id)
686
+ elif job_data['status'] == 'error' and (current_time - job_data.get('created_at', 0)) > 1800:
687
+ job_ids_to_remove.append(job_id)
688
+
689
+ for job_id in job_ids_to_remove:
690
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
691
+ try:
692
+ import shutil
693
+ if os.path.exists(output_dir):
694
+ shutil.rmtree(output_dir)
695
+ except Exception as e:
696
+ print(f"Error cleaning up job {job_id}: {str(e)}")
697
+
698
+ if job_id in processing_jobs:
699
+ del processing_jobs[job_id]
700
+
701
+ threading.Timer(300, cleanup_old_jobs).start()
702
 
703
+ @app.route('/model-info/<job_id>', methods=['GET'])
704
+ def model_info(job_id):
705
+ if job_id not in processing_jobs:
706
+ return jsonify({"error": "Model not found"}), 404
707
+
708
+ job = processing_jobs[job_id]
709
+
710
+ if job['status'] != 'completed':
711
+ return jsonify({
712
+ "status": job['status'],
713
+ "progress": job['progress'],
714
+ "error": job.get('error')
715
+ }), 200
716
+
717
+ output_dir = os.path.join(RESULTS_FOLDER, job_id)
718
+ model_stats = {}
719
+
720
+ if job['output_format'] == 'obj':
721
+ obj_path = os.path.join(output_dir, "model.obj")
722
+ zip_path = os.path.join(output_dir, "model.zip")
723
+ if os.path.exists(obj_path):
724
+ model_stats['obj_size'] = os.path.getsize(obj_path)
725
+ if os.path.exists(zip_path):
726
+ model_stats['package_size'] = os.path.getsize(zip_path)
727
+ else:
728
+ glb_path = os.path.join(output_dir, "model.glb")
729
+ if os.path.exists(glb_path):
730
+ model_stats['model_size'] = os.path.getsize(glb_path)
731
+
732
+ return jsonify({
733
+ "status": job['status'],
734
+ "model_format": job['output_format'],
735
+ "download_url": job['result_url'],
736
+ "preview_url": job['preview_url'],
737
+ "model_stats": model_stats,
738
+ "created_at": job.get('created_at'),
739
+ "completed_at": job.get('completed_at')
740
+ }), 200
741
 
742
+ @app.route('/', methods=['GET'])
743
+ def index():
744
+ return jsonify({
745
+ "message": "Multi-View Image to 3D API (DPT-Large + Depth Anything)",
746
+ "endpoints": [
747
+ "/convert",
748
+ "/progress/<job_id>",
749
+ "/download/<job_id>",
750
+ "/preview/<job_id>",
751
+ "/model-info/<job_id>"
752
+ ],
753
+ "parameters": {
754
+ "front": "Image file (required)",
755
+ "back": "Image file (required)",
756
+ "left": "Image file (optional)",
757
+ "right": "Image file (optional)",
758
+ "mesh_resolution": "Integer (50-120)",
759
+ "output_format": "obj or glb",
760
+ "detail_level": "low, medium, or high",
761
+ "texture_quality": "low, medium, or high"
762
+ },
763
+ "description": "Creates high-quality 3D models from multiple 2D images (front, back, left, right) using DPT-Large and Depth Anything."
764
+ }), 200
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
765
 
766
+ if __name__ == '__main__':
767
+ cleanup_old_jobs()
768
+ port = int(os.environ.get('PORT', 7860))
769
+ app.run(host='0.0.0.0', port=port)