diff --git a/.gitignore b/.gitignore index 82f927558a3dff0ea8c20858856e70779fd02c93..601d7e996d5202766a3304695e6792c8a0778c7c 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +.gradio/ \ No newline at end of file diff --git a/florence_sam/RAFT/__init__.py b/RAFT/__init__.py similarity index 100% rename from florence_sam/RAFT/__init__.py rename to RAFT/__init__.py diff --git a/florence_sam/RAFT/corr.py b/RAFT/corr.py similarity index 100% rename from florence_sam/RAFT/corr.py rename to RAFT/corr.py diff --git a/florence_sam/RAFT/datasets.py b/RAFT/datasets.py similarity index 100% rename from florence_sam/RAFT/datasets.py rename to RAFT/datasets.py diff --git a/florence_sam/RAFT/demo.py b/RAFT/demo.py similarity index 100% rename from florence_sam/RAFT/demo.py rename to RAFT/demo.py diff --git a/florence_sam/RAFT/extractor.py b/RAFT/extractor.py similarity index 100% rename from florence_sam/RAFT/extractor.py rename to RAFT/extractor.py diff --git a/florence_sam/RAFT/raft.py b/RAFT/raft.py similarity index 100% rename from florence_sam/RAFT/raft.py rename to RAFT/raft.py diff --git a/florence_sam/RAFT/update.py b/RAFT/update.py similarity index 100% rename from florence_sam/RAFT/update.py rename to RAFT/update.py diff --git a/florence_sam/RAFT/utils/__init__.py b/RAFT/utils/__init__.py similarity index 100% rename from florence_sam/RAFT/utils/__init__.py rename to RAFT/utils/__init__.py diff --git a/florence_sam/RAFT/utils/augmentor.py b/RAFT/utils/augmentor.py similarity index 100% rename from florence_sam/RAFT/utils/augmentor.py rename to RAFT/utils/augmentor.py diff --git a/florence_sam/RAFT/utils/flow_viz.py b/RAFT/utils/flow_viz.py similarity index 100% rename from florence_sam/RAFT/utils/flow_viz.py rename to RAFT/utils/flow_viz.py diff --git a/florence_sam/RAFT/utils/flow_viz_pt.py b/RAFT/utils/flow_viz_pt.py similarity index 100% rename from florence_sam/RAFT/utils/flow_viz_pt.py rename to RAFT/utils/flow_viz_pt.py diff --git a/florence_sam/RAFT/utils/frame_utils.py b/RAFT/utils/frame_utils.py similarity index 100% rename from florence_sam/RAFT/utils/frame_utils.py rename to RAFT/utils/frame_utils.py diff --git a/florence_sam/RAFT/utils/utils.py b/RAFT/utils/utils.py similarity index 100% rename from florence_sam/RAFT/utils/utils.py rename to RAFT/utils/utils.py diff --git a/florence_sam/app.py b/app.py similarity index 59% rename from florence_sam/app.py rename to app.py index 53b5f62b820dcba39ec074b483ba3bf91341b05a..e0f36d7541185777fdc14899faf3cb668e93bc20 100644 --- a/florence_sam/app.py +++ b/app.py @@ -2,26 +2,25 @@ from main import infer import moviepy.editor as mp import gradio as gr import os +import tempfile def pre_processor(video_path, scale_factor, prompt, crop_duration): video = mp.VideoFileClip(video_path) - cropped_video = video.subclip(0, min(crop_duration, video.duration)) - temp_output = "cropped_video.mp4" - cropped_video.write_videofile(temp_output, codec="libx264") + with tempfile.NamedTemporaryFile(suffix='.mp4', delete=False) as temp_file: + temp_output = temp_file.name + cropped_video.write_videofile(temp_output, codec="libx264") - output = infer(temp_output, scale_factor, prompt) + output = infer(temp_output, scale_factor, prompt) - # Clean up temporary files - if os.path.exists(temp_output): - os.remove(temp_output) + # Clean up temporary file + os.unlink(temp_output) return output - demo = gr.Interface( - title="Text based video inpainting", + title="Text Based Video Inpainting 🔥 (SAM2+Florance2+ProPainter)", fn=pre_processor, inputs=[ gr.Video(label="Upload Video"), @@ -32,4 +31,4 @@ demo = gr.Interface( outputs="video" ) -demo.launch() +demo.launch(server_port=7555) diff --git a/florence_sam/checkpoints/sam2_hiera_base_plus.pt b/checkpoints/sam2_hiera_base_plus.pt similarity index 100% rename from florence_sam/checkpoints/sam2_hiera_base_plus.pt rename to checkpoints/sam2_hiera_base_plus.pt diff --git a/florence_sam/checkpoints/sam2_hiera_large.pt b/checkpoints/sam2_hiera_large.pt similarity index 100% rename from florence_sam/checkpoints/sam2_hiera_large.pt rename to checkpoints/sam2_hiera_large.pt diff --git a/florence_sam/checkpoints/sam2_hiera_small.pt b/checkpoints/sam2_hiera_small.pt similarity index 100% rename from florence_sam/checkpoints/sam2_hiera_small.pt rename to checkpoints/sam2_hiera_small.pt diff --git a/florence_sam/checkpoints/sam2_hiera_tiny.pt b/checkpoints/sam2_hiera_tiny.pt similarity index 100% rename from florence_sam/checkpoints/sam2_hiera_tiny.pt rename to checkpoints/sam2_hiera_tiny.pt diff --git a/florence_sam/configs/__init__.py b/configs/__init__.py similarity index 100% rename from florence_sam/configs/__init__.py rename to configs/__init__.py diff --git a/florence_sam/configs/sam2_hiera_b+.yaml b/configs/sam2_hiera_b+.yaml similarity index 100% rename from florence_sam/configs/sam2_hiera_b+.yaml rename to configs/sam2_hiera_b+.yaml diff --git a/florence_sam/configs/sam2_hiera_l.yaml b/configs/sam2_hiera_l.yaml similarity index 100% rename from florence_sam/configs/sam2_hiera_l.yaml rename to configs/sam2_hiera_l.yaml diff --git a/florence_sam/configs/sam2_hiera_s.yaml b/configs/sam2_hiera_s.yaml similarity index 100% rename from florence_sam/configs/sam2_hiera_s.yaml rename to configs/sam2_hiera_s.yaml diff --git a/florence_sam/configs/sam2_hiera_t.yaml b/configs/sam2_hiera_t.yaml similarity index 100% rename from florence_sam/configs/sam2_hiera_t.yaml rename to configs/sam2_hiera_t.yaml diff --git a/florence_sam/configs/train_flowcomp.json b/configs/train_flowcomp.json similarity index 100% rename from florence_sam/configs/train_flowcomp.json rename to configs/train_flowcomp.json diff --git a/florence_sam/configs/train_propainter.json b/configs/train_propainter.json similarity index 100% rename from florence_sam/configs/train_propainter.json rename to configs/train_propainter.json diff --git a/florence_sam/core/dataset.py b/core/dataset.py similarity index 100% rename from florence_sam/core/dataset.py rename to core/dataset.py diff --git a/florence_sam/core/dist.py b/core/dist.py similarity index 100% rename from florence_sam/core/dist.py rename to core/dist.py diff --git a/florence_sam/core/loss.py b/core/loss.py similarity index 100% rename from florence_sam/core/loss.py rename to core/loss.py diff --git a/florence_sam/core/lr_scheduler.py b/core/lr_scheduler.py similarity index 100% rename from florence_sam/core/lr_scheduler.py rename to core/lr_scheduler.py diff --git a/florence_sam/core/metrics.py b/core/metrics.py similarity index 100% rename from florence_sam/core/metrics.py rename to core/metrics.py diff --git a/florence_sam/core/prefetch_dataloader.py b/core/prefetch_dataloader.py similarity index 100% rename from florence_sam/core/prefetch_dataloader.py rename to core/prefetch_dataloader.py diff --git a/florence_sam/core/trainer.py b/core/trainer.py similarity index 100% rename from florence_sam/core/trainer.py rename to core/trainer.py diff --git a/florence_sam/core/trainer_flow_w_edge.py b/core/trainer_flow_w_edge.py similarity index 100% rename from florence_sam/core/trainer_flow_w_edge.py rename to core/trainer_flow_w_edge.py diff --git a/florence_sam/core/utils.py b/core/utils.py similarity index 100% rename from florence_sam/core/utils.py rename to core/utils.py diff --git a/florence_sam/datasets/davis/test.json b/datasets/davis/test.json similarity index 100% rename from florence_sam/datasets/davis/test.json rename to datasets/davis/test.json diff --git a/florence_sam/datasets/davis/train.json b/datasets/davis/train.json similarity index 100% rename from florence_sam/datasets/davis/train.json rename to datasets/davis/train.json diff --git a/florence_sam/datasets/youtube-vos/test.json b/datasets/youtube-vos/test.json similarity index 100% rename from florence_sam/datasets/youtube-vos/test.json rename to datasets/youtube-vos/test.json diff --git a/florence_sam/datasets/youtube-vos/train.json b/datasets/youtube-vos/train.json similarity index 100% rename from florence_sam/datasets/youtube-vos/train.json rename to datasets/youtube-vos/train.json diff --git a/florence_sam/experiment.ipynb b/experiment.ipynb similarity index 100% rename from florence_sam/experiment.ipynb rename to experiment.ipynb diff --git a/florence_sam/florancesam_pipeline.py b/florancesam_pipeline.py similarity index 87% rename from florence_sam/florancesam_pipeline.py rename to florancesam_pipeline.py index 9aaec2b787a05cbff71ffe6c54200b9e1c3b526f..9d865364b9881ad5ff34b38542df55549714aa9b 100644 --- a/florence_sam/florancesam_pipeline.py +++ b/florancesam_pipeline.py @@ -52,15 +52,15 @@ class VideoProcessor: self.scale_factor = scale_factor # Process video based on the prompt - output_video_path, session_path = self._process_prompt(video_path, prompt) + output_video_path, session_path, input_frames_dir, output_directory_path = self._process_prompt(video_path, prompt) # Create frames from the output video - fps = self._create_frames(output_video_path, os.path.join(session_path, "output_frames")) + fps = self._create_frames(output_video_path, output_directory_path) # Delete the output video os.remove(output_video_path) - return session_path, fps + return session_path, fps, input_frames_dir, output_directory_path def _create_frames(self, video_path, output_dir): create_directory(output_dir) @@ -119,10 +119,14 @@ class VideoProcessor: # Generate unique name for video processing name = generate_unique_name() - session_path = os.path.join("tmp", name) - create_directory(session_path) - frame_directory_path = os.path.join(session_path, "input_frames") - create_directory(frame_directory_path) + # session_path = os.path.join("tmp", name) + # create_directory(session_path) + # frame_directory_path = os.path.join(session_path, "input_frames") + # create_directory(frame_directory_path) + import tempfile + session_path = tempfile.mkdtemp(prefix="video_processing_") + frame_directory_path = tempfile.mkdtemp(prefix="input_frames_", dir=session_path) + output_directory_path = tempfile.mkdtemp(prefix="output_frames_", dir=session_path) frames_sink = sv.ImageSink( target_dir_path=frame_directory_path, @@ -157,7 +161,8 @@ class VideoProcessor: ) # Create output video path - output_video_path = os.path.join("tmp", f"{name}.mp4") + # output_video_path = os.path.join("tmp", f"{name}.mp4") + output_video_path = os.path.join(session_path, f"{name}.mp4") frames_generator = sv.get_video_frames_generator(video_path) masks_generator = self.sam_video_model.propagate_in_video(inference_state) @@ -185,7 +190,7 @@ class VideoProcessor: annotated_frame = (annotated_frame > 0).astype(np.uint8) * 255 sink.write_frame(annotated_frame) - return output_video_path, session_path + return output_video_path, session_path, frame_directory_path, output_directory_path #Example usage # output_video = video_processor.process_video( diff --git a/florence_sam/README.md b/florence_sam/README.md deleted file mode 100644 index c951c84e1aed32d1277c92c1d36854dbbbd7824b..0000000000000000000000000000000000000000 --- a/florence_sam/README.md +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:031c6f979f28b8b0f3b42af389fd1f741046ca89de9bc05a33912db7e30e8741 -size 258 diff --git a/florence_sam/requirements.txt b/florence_sam/requirements.txt deleted file mode 100644 index 38bdf8eef5e0eabf2a2eb5c77353ed5e46c8bf26..0000000000000000000000000000000000000000 --- a/florence_sam/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -tqdm -einops -spaces -timm -transformers -samv2 -gradio -supervision -opencv-python -pytest -imageio -moviepy \ No newline at end of file diff --git a/florence_sam/inference_propainter.py b/inference_propainter.py similarity index 100% rename from florence_sam/inference_propainter.py rename to inference_propainter.py diff --git a/florence_sam/main.py b/main.py similarity index 73% rename from florence_sam/main.py rename to main.py index e485b7084e4b94e7f3c21bdd9f08a2ec80800ce5..f659f9f40e77e07d7392c651fdf980a556d011b3 100644 --- a/florence_sam/main.py +++ b/main.py @@ -3,6 +3,7 @@ import shutil import torch from propainter_pipeline import process_video from florancesam_pipeline import VideoProcessor +import tempfile video_processor = VideoProcessor() video_processor._enable_mixed_precision() @@ -12,7 +13,7 @@ def infer(video_path, scale_factor, prompt): video_processor._enable_mixed_precision() # -------------------------------------------------------------------------------- print("Processing video with FlorenceSam...") - session_path, fps = video_processor.process_video( + session_path, fps, input_frames_dir, output_frames_dir = video_processor.process_video( video_path=video_path, scale_factor=scale_factor, prompt=prompt @@ -26,13 +27,14 @@ def infer(video_path, scale_factor, prompt): torch.cuda.empty_cache() print("Processing video with ProPainter...") - process_video(video=os.path.join(session_path, "input_frames"), mask=os.path.join(session_path, "output_frames"), save_fps=int(fps), fp16=True) + result_path = tempfile.mkdtemp() + inpainted_video = process_video(video=input_frames_dir, mask=output_frames_dir, save_fps=int(fps), fp16=True, output=result_path) - # remove intermediate files - shutil.rmtree("tmp") + shutil.rmtree(input_frames_dir) + shutil.rmtree(output_frames_dir) torch.cuda.empty_cache() - - return "results/input_frames/inpaint_out.mp4" + + return inpainted_video if __name__ == "__main__": infer("/home/ubuntu/ahmedghani/clip-07-camera-2.mp4", 0.5, "players, basketball, rim, players shadow") \ No newline at end of file diff --git a/florence_sam/model/__init__.py b/model/__init__.py similarity index 100% rename from florence_sam/model/__init__.py rename to model/__init__.py diff --git a/florence_sam/model/canny/canny_filter.py b/model/canny/canny_filter.py similarity index 100% rename from florence_sam/model/canny/canny_filter.py rename to model/canny/canny_filter.py diff --git a/florence_sam/model/canny/filter.py b/model/canny/filter.py similarity index 100% rename from florence_sam/model/canny/filter.py rename to model/canny/filter.py diff --git a/florence_sam/model/canny/gaussian.py b/model/canny/gaussian.py similarity index 100% rename from florence_sam/model/canny/gaussian.py rename to model/canny/gaussian.py diff --git a/florence_sam/model/canny/kernels.py b/model/canny/kernels.py similarity index 100% rename from florence_sam/model/canny/kernels.py rename to model/canny/kernels.py diff --git a/florence_sam/model/canny/sobel.py b/model/canny/sobel.py similarity index 100% rename from florence_sam/model/canny/sobel.py rename to model/canny/sobel.py diff --git a/florence_sam/model/misc.py b/model/misc.py similarity index 100% rename from florence_sam/model/misc.py rename to model/misc.py diff --git a/florence_sam/model/modules/base_module.py b/model/modules/base_module.py similarity index 100% rename from florence_sam/model/modules/base_module.py rename to model/modules/base_module.py diff --git a/florence_sam/model/modules/deformconv.py b/model/modules/deformconv.py similarity index 100% rename from florence_sam/model/modules/deformconv.py rename to model/modules/deformconv.py diff --git a/florence_sam/model/modules/flow_comp_raft.py b/model/modules/flow_comp_raft.py similarity index 100% rename from florence_sam/model/modules/flow_comp_raft.py rename to model/modules/flow_comp_raft.py diff --git a/florence_sam/model/modules/flow_loss_utils.py b/model/modules/flow_loss_utils.py similarity index 100% rename from florence_sam/model/modules/flow_loss_utils.py rename to model/modules/flow_loss_utils.py diff --git a/florence_sam/model/modules/sparse_transformer.py b/model/modules/sparse_transformer.py similarity index 100% rename from florence_sam/model/modules/sparse_transformer.py rename to model/modules/sparse_transformer.py diff --git a/florence_sam/model/modules/spectral_norm.py b/model/modules/spectral_norm.py similarity index 100% rename from florence_sam/model/modules/spectral_norm.py rename to model/modules/spectral_norm.py diff --git a/florence_sam/model/propainter.py b/model/propainter.py similarity index 100% rename from florence_sam/model/propainter.py rename to model/propainter.py diff --git a/florence_sam/model/recurrent_flow_completion.py b/model/recurrent_flow_completion.py similarity index 100% rename from florence_sam/model/recurrent_flow_completion.py rename to model/recurrent_flow_completion.py diff --git a/florence_sam/model/vgg_arch.py b/model/vgg_arch.py similarity index 100% rename from florence_sam/model/vgg_arch.py rename to model/vgg_arch.py diff --git a/florence_sam/propainter_pipeline.py b/propainter_pipeline.py similarity index 99% rename from florence_sam/propainter_pipeline.py rename to propainter_pipeline.py index d853a7b61d18272345fa18851b074674b63001fe..781e115cc6d024cf41f3b91473c44eba8f9985d5 100644 --- a/florence_sam/propainter_pipeline.py +++ b/propainter_pipeline.py @@ -286,4 +286,6 @@ def process_video( print(f'\nAll results are saved in {save_root}') - torch.cuda.empty_cache() \ No newline at end of file + torch.cuda.empty_cache() + + return os.path.join(save_root, 'inpaint_out.mp4') \ No newline at end of file diff --git a/florence_sam/propainter_utils/download_util.py b/propainter_utils/download_util.py similarity index 100% rename from florence_sam/propainter_utils/download_util.py rename to propainter_utils/download_util.py diff --git a/florence_sam/propainter_utils/file_client.py b/propainter_utils/file_client.py similarity index 100% rename from florence_sam/propainter_utils/file_client.py rename to propainter_utils/file_client.py diff --git a/florence_sam/propainter_utils/flow_util.py b/propainter_utils/flow_util.py similarity index 100% rename from florence_sam/propainter_utils/flow_util.py rename to propainter_utils/flow_util.py diff --git a/florence_sam/propainter_utils/img_util.py b/propainter_utils/img_util.py similarity index 100% rename from florence_sam/propainter_utils/img_util.py rename to propainter_utils/img_util.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..cc897af446441b2e299302f008c9d46c342ad269 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +tqdm +einops==0.8.0 +spaces +timm==1.0.14 +transformers==4.48.1 +samv2==0.0.4 +gradio +supervision==0.25.1 +opencv-python==4.11.0.86 +pytest==8.3.4 +imageio==2.37.0 +moviepy==1.0.3 \ No newline at end of file diff --git a/florence_sam/scripts/compute_flow.py b/scripts/compute_flow.py similarity index 100% rename from florence_sam/scripts/compute_flow.py rename to scripts/compute_flow.py diff --git a/florence_sam/scripts/evaluate_flow_completion.py b/scripts/evaluate_flow_completion.py similarity index 100% rename from florence_sam/scripts/evaluate_flow_completion.py rename to scripts/evaluate_flow_completion.py diff --git a/florence_sam/scripts/evaluate_propainter.py b/scripts/evaluate_propainter.py similarity index 100% rename from florence_sam/scripts/evaluate_propainter.py rename to scripts/evaluate_propainter.py diff --git a/florence_sam/utils/__init__.py b/utils/__init__.py similarity index 100% rename from florence_sam/utils/__init__.py rename to utils/__init__.py diff --git a/florence_sam/utils/florence.py b/utils/florence.py similarity index 98% rename from florence_sam/utils/florence.py rename to utils/florence.py index 9887f4406b5c022d7a3ea36437e3119ab99fd124..73d141a48203ae006fa8dafc4721e90e1c5e1239 100644 --- a/florence_sam/utils/florence.py +++ b/utils/florence.py @@ -20,7 +20,7 @@ def fixed_get_imports(filename: Union[str, os.PathLike]) -> list[str]: if not str(filename).endswith("/modeling_florence2.py"): return get_imports(filename) imports = get_imports(filename) - imports.remove("flash_attn") + # imports.remove("flash_attn") return imports diff --git a/florence_sam/utils/modes.py b/utils/modes.py similarity index 100% rename from florence_sam/utils/modes.py rename to utils/modes.py diff --git a/florence_sam/utils/sam.py b/utils/sam.py similarity index 100% rename from florence_sam/utils/sam.py rename to utils/sam.py diff --git a/florence_sam/utils/video.py b/utils/video.py similarity index 100% rename from florence_sam/utils/video.py rename to utils/video.py diff --git a/florence_sam/videos/clip-07-camera-1.mp4 b/videos/clip-07-camera-1.mp4 similarity index 100% rename from florence_sam/videos/clip-07-camera-1.mp4 rename to videos/clip-07-camera-1.mp4 diff --git a/florence_sam/weights/ProPainter.pth b/weights/ProPainter.pth similarity index 100% rename from florence_sam/weights/ProPainter.pth rename to weights/ProPainter.pth diff --git a/florence_sam/weights/README.md b/weights/README.md similarity index 100% rename from florence_sam/weights/README.md rename to weights/README.md diff --git a/florence_sam/weights/i3d_rgb_imagenet.pt b/weights/i3d_rgb_imagenet.pt similarity index 100% rename from florence_sam/weights/i3d_rgb_imagenet.pt rename to weights/i3d_rgb_imagenet.pt diff --git a/florence_sam/weights/raft-things.pth b/weights/raft-things.pth similarity index 100% rename from florence_sam/weights/raft-things.pth rename to weights/raft-things.pth diff --git a/florence_sam/weights/recurrent_flow_completion.pth b/weights/recurrent_flow_completion.pth similarity index 100% rename from florence_sam/weights/recurrent_flow_completion.pth rename to weights/recurrent_flow_completion.pth